#  Copyright (c) 2026 Cisco Systems, Inc. and its affiliates
#  SPDX-License-Identifier: Apache-2.0
$schema: "http://json-schema.org/draft-07/schema#"
$id: "mas-lab/experiment/v1"
title: "mas-lab experiment manifest (v1)"
description: >
  Schema for experiment.yaml — declarative batch benchmark configuration.
  Top-level key must be 'experiment:'.  All relative paths are resolved from
  the directory containing this file.

  CLI vocabulary (canonical): application (MAS binding), scenarios, dataset,
  run, item, scenario, post (experiment-level, --depth exp).

  Properties marked deprecated: true still validate so old files load;
  mas-lab validate warns on them (applications, mas). Removed keys
  (pipeline_bind, pipeline, plots, flavours, output_dir) are rejected by
  additionalProperties: false.

type: object
required: [experiment]
additionalProperties: false

properties:
  experiment:
    type: object
    additionalProperties: false
    description: "Batch benchmark configuration — scenarios × dataset × runs."

    properties:
      # ── Identity ──────────────────────────────────────────────────────────
      name:
        type: string
        minLength: 1
        description: "Unique experiment identifier (used in output paths and reports)."

      description:
        type: string
        description: "Free-text description shown in benchmark reports."

      metadata:
        type: object
        additionalProperties: true
        description: "Optional reproducibility metadata (model, dataset version, notes)."

      output_schema:
        type: object
        additionalProperties: false
        deprecated: true
        x-replaced-by: "plotnine/config.df_schema (and compute_ci output paths)"
        description: >
          DEPRECATED. Prefer plot-step df_schema and the files those steps write.
          Still accepted as a warn-only post-run checklist.
        properties:
          required_files:
            type: array
            items:
              type: string
          required_columns:
            type: object
            additionalProperties:
              type: array
              items:
                type: string

      default_flavour:
        type: string
        description: >
          Runtime flavour name resolved via library-standard
          (mas.lab.flavour.resolve).

      model:
        type: string
        description: >
          Shorthand for models.main. Experiment default LLM (LiteLLM id, or
          `any`). Agents/MAS that omit spec.models or set model: any inherit
          this. `any` (and omitted) means this experiment does not pin a
          provider id — inherit the application spec.models, then local
          config.yaml defaults.model. models.main wins when both are set.
          See docs/manifests/summarization.md.

      models:
        type: object
        additionalProperties:
          type: string
        description: >
          Slot map matching Agent/MAS spec.models[].id, plus `judge`.
          `main` is the turn default (same as scalar model). `summarizer`
          defaults the summary LLM when the agent omitted params.model.
          `judge` defaults eval_mce; evaluation.model still overrides the
          judge only. Values are LiteLLM ids or `any`.

      default_infra:
        type: string
        description: "Default infra bundle name for service/codec pipeline steps."

      # ── MAS under test (CLI has no --application depth; this is the MAS binding) ─
      application:
        description: >
          The MAS or agent under test ({app|manifest, configs_dir}).
          Using this key as a pipeline level (pre/post without app/manifest) is
          deprecated; use experiment-level post: (CLI --depth exp).
        oneOf:
          - $ref: "#/$defs/mas_binding"
          - allOf:
              - $ref: "#/$defs/level_section"
              - deprecated: true
                description: "DEPRECATED. Use experiment-level post: (CLI --depth exp)."

      applications:
        type: array
        minItems: 1
        deprecated: true
        x-replaced-by: "application: {app|manifest, configs_dir}"
        description: >
          DEPRECATED. Use application: {app|manifest, configs_dir}.
          Still accepted as a one-item list so existing files load.
        items:
          oneOf:
            - type: string
              minLength: 1
              description: >
                Library + app identifier. The applications: attribute implies
                kind MAS (or a standalone Agent app). Example:
                example-library:my-app@v2. Slash after LIBRARY: is a path,
                not an id (example-library:apps/my-app/v2).
              pattern: "^[a-z0-9][a-z0-9-]*:[^\\s]+$"
            - type: object
              additionalProperties: false
              description: "Pointer to a MAS use-case (manifest path, app name, or configs directory)."
              properties:
                manifest:
                  type: string
                  description: >
                    Path or library:app identifier resolving to mas.yaml
                    (e.g. example-library:my-app@v2).
                app:
                  type: string
                  description: >
                    Application catalog id, optionally library-qualified
                    (example-library:my-app@v2 or my-app@v2).
                configs_dir:
                  type: string
                  description: "Path to the overlays directory."
                base_scenario:
                  type: string
                  default: "baseline"
                  description: "Default scenario used as reference/baseline."
              anyOf:
                - required: [manifest]
                - required: [app]

      mas:
        deprecated: true
        description: >
          DEPRECATED. Alias of application: {app|manifest, configs_dir}.
          Loaders still accept mas: {manifest|app, configs_dir} (and
          lab.mas in lab-config.yaml) and emit a one-shot warning.
          Replace with application: {app|manifest, configs_dir}.
          See https://ioc-project.github.io/mas-lab/manifests/experiment/#applications
        oneOf:
          - type: object
            additionalProperties: true
          - type: array
            minItems: 1
          - type: string
            minLength: 1

      # ── Scenarios ─────────────────────────────────────────────────────────
      scenarios:
        type: array
        description: "Ordered list of scenarios to benchmark."
        items:
          type: object
          required: [id]
          additionalProperties: false
          properties:
            id:
              type: string
              minLength: 1
              description: "Unique scenario identifier within this experiment. Also used as overlay name when overlays is absent."
            overlays:
              type: object
              additionalProperties: false
              description: "Layered overlay stacks (logic / control / infra)."
              required: [logic, control, infra]
              properties:
                logic:
                  type: array
                  items:
                    oneOf:
                      - type: string
                        minLength: 1
                      - type: object
                        required: [ref]
                        additionalProperties: false
                        properties:
                          ref:
                            type: string
                            minLength: 1
                  default: []
                control:
                  type: array
                  items:
                    oneOf:
                      - type: string
                        minLength: 1
                      - type: object
                        required: [ref]
                        additionalProperties: false
                        properties:
                          ref:
                            type: string
                            minLength: 1
                  default: []
                infra:
                  type: array
                  items:
                    oneOf:
                      - type: string
                        minLength: 1
                      - type: object
                        required: [ref]
                        additionalProperties: false
                        properties:
                          ref:
                            type: string
                            minLength: 1
                  default: []
            description:
              type: string
            tags:
              type: array
              items:
                type: string
              description: "Labels for grouping/filtering (e.g. ['reference', 'baseline'])."
            flavour:
              type: string
              description: "Per-scenario flavour override (name or path)."

      checkpoints:
        type: array
        minItems: 1
        description: >-
          Optional starting-state axis crossed with scenarios, dataset items,
          and run.n_runs. Omitted means one implicit fresh start.
        items:
          type: object
          required: [id]
          additionalProperties: false
          properties:
            id:
              type: string
              minLength: 1
              pattern: "^[A-Za-z0-9_-]+$"
            path:
              type: string
              minLength: 1
          allOf:
            - if:
                properties:
                  id: { const: none }
                required: [id]
              then:
                not:
                  required: [path]
            - if:
                properties:
                  id: { not: { const: none } }
                required: [id]
              then:
                required: [path]

      # ── Dataset ───────────────────────────────────────────────────────────
      dataset:
        type: object
        additionalProperties: false
        description: >-
          Benchmark dataset reference.  Resolved via mas.registry in priority
          order: explicit path > cross-app lookup > global mas.datasets registry
          > local datasets/ folder fallback.
        properties:
          name:
            type: string
            description: >-
              Dataset name — resolved via the global mas.datasets registry, or
              as a local-folder fallback ({experiment_dir}/datasets/{name}.yaml).
          app:
            type: string
            description: >-
              Legacy filesystem hint: look under that app for datasets/<name>.
              App-specific datasets now live at apps/<app>/vN/datasets/<name>/
              and declare spec.app on the Dataset itself. Prefer dataset.name
              as a catalog id.
          path:
            type: string
            description: >-
              Explicit relative or absolute path to the dataset YAML.  Overrides
              name/app resolution.
          locator:
            type: string
            description: >-
              Manifest library scheme (e.g. ``samples``) or package name.  Resolves
              ``name`` via that library's ``library.yaml`` dataset catalog.
          filter:
            type: object
            additionalProperties: true
            description: "Optional filter applied to dataset items before benchmarking."
          limit:
            type: integer
            minimum: 1
            description: >
              Use the first N items of the Dataset. Prefer this over shipping
              a reduced copy (for example a ``-100`` sidecar). Smoke and CI
              set a small limit; full lab runs omit it.
          source:
            type: object
            additionalProperties: false
            description: >
              Optional external-dataset pointer (same shape as Dataset
              spec.source). Merged over the Dataset manifest's spec.source
              at load time (e.g. a tighter limit or different split).
            properties:
              kind:
                type: string
                enum: [huggingface, jsonl, csv, glob, pickle]
              id:
                type: string
              split:
                type: string
              path:
                type: string
              map:
                type: object
                additionalProperties: true
              config:
                type: string
              name:
                type: string
              limit:
                type: integer
                minimum: 1

      # ── Evaluation ────────────────────────────────────────────────────────
      evaluation:
        type: object
        required: [method]
        additionalProperties: false
        description: "Automatic evaluation spec applied to benchmark outputs."
        properties:
          method:
            type: string
            enum: [user_emulation, llm_judge, metrics, trace_only]
            description: "Evaluation strategy. trace_only = collect trajectories without scoring."
          model:
            type: string
            description: >
              LLM-as-judge model for eval_mce (and other llm_judge steps).
              Overrides experiment.models.judge / experiment.model /
              application spec.models for the judge only. Omit to inherit
              those defaults (`any` resolves from local config.yaml). A
              per-step eval_mce config.model still wins. See
              docs/manifests/summarization.md#mce-judge-model.
          config:
            type: object
            additionalProperties: true
            description: >
              Method-specific configuration. config.model is an alias of
              evaluation.model (evaluation.model wins when both are set).

      # ── Output ────────────────────────────────────────────────────────────
      trace_cache_dir:
        type: string
        description: >
          Override the global trace-cache directory for this experiment.
          Falls back to env var MAS_TRACE_CACHE, then $XDG_CACHE_HOME/mas/traces
          (see docs/user-config.md).
      trace_cache_portable:
        type: boolean
        default: false
        description: >
          Leave the resolved LLM infra (endpoint, model mapping) out of the
          trace-cache key, so a published cache hits on any endpoint. Agent
          model names still enter the key. Use it for caches you publish.

      # ── Design (what is compared) ─────────────────────────────────────────
      design:
        $ref: "#/$defs/experiment_design"
        description: >
          Experimental design mode and cartesian guard. Changing these fields
          changes the meaning of a result cell. Replication count lives under
          run.n_runs, not here.

      # ── Schedule (how the bench walks the matrix) ─────────────────────────
      schedule:
        $ref: "#/$defs/experiment_schedule"
        description: >
          Bench progression only — parallelism, timeouts, ordering, runner.
          Never changes what a result cell means, only wall time / resource
          usage / failure modes.

      # ── Bench emulation (held constant unless a declared factor) ──────────
      bench_emulation:
        $ref: "#/$defs/bench_emulation"
        description: >
          Mock/replay/trace-cache posture for this study. Distinct from
          workspace infra_refs / RuntimeEngine LLM cache. When infra is an
          intentional factor, use scenario overlays.infra instead.

      replay:
        type: object
        additionalProperties: false
        description: "Multi-turn dataset replay (design + dataset, not schedule)."
        properties:
          mode:
            type: string
            enum: [scripted, cache-and-diverge]
            default: scripted
          hitl_source:
            type: string
            enum: [dataset, recording]
            default: dataset
          recording_path:
            type: string
          diverge_from_cache:
            type: boolean
            default: true

      # ── Execution (deprecated dual-read shim) ─────────────────────────────
      execution:
        type: object
        additionalProperties: false
        deprecated: true
        description: >
          DEPRECATED. Split into experiment.design, experiment.schedule, and
          experiment.bench_emulation. Still accepted for one release (dual-read
          loader emits a deprecation warning). Not agent spec.execution (removed).
        properties:
          parallel_scenarios:
            type: integer
            minimum: 1
            default: 4
          timeout:
            type: integer
            minimum: 1
            default: 300
            description: "Per-run timeout in seconds."
          max_attempts:
            type: integer
            minimum: 1
            default: 3
            description: >
              Re-issue a failed scenario×item×run this many times (including
              the first try). Independent of run.n_runs. Default 3 so an
              LLM HTTP failure can recover after in-call retries. Set 1 to disable.
          retry_backoff_s:
            type: number
            minimum: 0
            default: 2.0
            description: >
              Base delay in seconds before the first lab-level reattempt;
              doubles on each subsequent reattempt.
          pause_between_runs:
            type: number
            minimum: 0
            default: 1.0
          strategy:
            type: string
            enum: [coverage, depth]
            default: "coverage"
            deprecated: true
            description: "Deprecated alias of schedule.ordering."
          ordering:
            type: string
            enum: [coverage, depth]
          runner:
            type: string
            minLength: 1
            default: native
            description: >
              Lab adapter id (mas.lab.runners entry point). Optional override —
              when omitted, inferred from application.app / mas.manifest
              spec.framework.default_adapter or agent spec.framework_adapter.
          reset_state:
            type: boolean
          n_runs:
            type: integer
            minimum: 1
            deprecated: true
            description: "Deprecated; use run.n_runs (design replication)."
          emulation:
            type: object
            additionalProperties: true
          design:
            $ref: "#/$defs/experiment_design"
          replay:
            type: object
            additionalProperties: true
          faults:
            type: object
            additionalProperties: true

      # ── Artifacts ─────────────────────────────────────────────────────────
      artifacts:
        $ref: "#/$defs/artifact_declarations"

      # ── Pipeline hooks (CLI --depth exp|scenario|item|run) ──────────────
      run:
        $ref: "#/$defs/run_level_section"
      item:
        $ref: "#/$defs/level_section"
        description: >
          Hooks for one dataset item after all its runs (CLI --item / --depth item).
      test:
        $ref: "#/$defs/level_section"
        deprecated: true
        x-replaced-by: "item:"
        description: "DEPRECATED. Use item: (matches CLI --item)."
      scenario:
        $ref: "#/$defs/level_section"
        description: "Hooks after all items in a scenario (CLI --scenario)."
      pre:
        type: array
        description: "Experiment-level pre hooks (CLI --depth exp)."
        items:
          $ref: "#/$defs/level_pipeline_entry"
      post:
        type: array
        description: >
          Experiment-level post hooks after every scenario completes
          (CLI --depth exp). Preferred over the deprecated application.post.
        items:
          $ref: "#/$defs/level_pipeline_entry"

      ui:
        type: object
        additionalProperties: false
        description: "Demo UI configuration hints (used by 'mas-lab demo', ignored in batch mode)."
        properties:
          port:
            type: integer
            minimum: 1
            maximum: 65535
            description: "HTTP port for the demo server."
          mode:
            type: string
            enum: [interactive, automated]
            default: "interactive"
          layout:
            type: string
            enum: [dag, grid, default]
            default: "default"
          node_positions:
            type: object
            additionalProperties:
              type: object
              additionalProperties: false
              required: [x, y]
              properties:
                x:
                  type: number
                y:
                  type: number
            description: "Per-agent canvas coordinates {agent_id: {x, y}}."

    required: [name]
    anyOf:
      - required: [application]
      - required: [applications]
      - required: [mas]

# ── Shared definitions ────────────────────────────────────────────────────────
$defs:
  mas_binding:
    type: object
    additionalProperties: false
    description: "Pointer to a MAS use-case (manifest path, app name, or configs directory)."
    properties:
      manifest:
        type: string
        description: "Path to mas.yaml or agent.yaml."
      app:
        type: string
        description: Registered app name (mas.apps registry → mas.yaml).
      configs_dir:
        type: string
        description: "Path to the overlays directory."
      base_scenario:
        type: string
        default: "baseline"
        description: "Default scenario used as reference/baseline."
    anyOf:
      - required: [manifest]
      - required: [app]

  experiment_design:
    type: object
    additionalProperties: false
    description: >
      How to pick cells from scenarios × dataset items. cartesian = full
      product; coupled = listed pairs; one_factor = vary one of those two
      axes. Other study axes (checkpoints, overlays, models, infra) are
      not combinators here.
    properties:
      mode:
        type: string
        enum: [cartesian, coupled, one_factor]
        default: cartesian
      max_executions:
        type: integer
        minimum: 1
        description: "Fail when scenarios × items × n_runs exceeds this limit."
      couplings:
        type: array
        minItems: 1
        items:
          type: object
          additionalProperties: false
          required: [scenario]
          properties:
            scenario:
              type: string
            items:
              type: array
      vary:
        type: object
        additionalProperties: false
        required: [axis, values]
        description: "one_factor only: which scenario/item axis to sweep."
        properties:
          axis:
            type: string
            enum: [scenario, item]
          values:
            type: array
            minItems: 1
      pin_scenarios:
        type: array
        minItems: 1
        items:
          type: string
        description: "one_factor: scenarios held constant. Omit to use every scenario."
      pin_items:
        type: array
        minItems: 1
        description: "one_factor: item ids held constant. Omit to use every item."
    allOf:
      - if:
          properties:
            mode: { const: one_factor }
          required: [mode]
        then:
          required: [vary]
        else:
          not:
            anyOf:
              - required: [vary]
              - required: [pin_scenarios]
              - required: [pin_items]
      - if:
          properties:
            mode: { const: coupled }
          required: [mode]
        then:
          required: [couplings]
        else:
          not:
            required: [couplings]

  experiment_schedule:
    type: object
    additionalProperties: false
    description: "Bench progression — parallelism, timeouts, ordering, runner."
    properties:
      parallel_scenarios:
        type: integer
        minimum: 1
        default: 4
        description: "Max concurrent MAS runs."
      timeout:
        type: integer
        minimum: 1
        default: 300
        description: "Per-run timeout in seconds."
      max_attempts:
        type: integer
        minimum: 1
        default: 3
        description: >
          Re-issue a failed scenario×item×run this many times (including
          the first try). Independent of run.n_runs. Default 3. Set 1 to disable.
      retry_backoff_s:
        type: number
        minimum: 0
        default: 2.0
        description: >
          Base delay in seconds before the first lab-level reattempt;
          doubles on each subsequent reattempt.
      pause_between_runs:
        type: number
        minimum: 0
        default: 1.0
        description: "Sleep in seconds between runs (let resources settle)."
      ordering:
        type: string
        enum: [coverage, depth]
        default: "coverage"
        description: >
          Walk order.
          'coverage' (default): breadth-first — one round across all conditions before the next.
          'depth': depth-first — finish all n_runs for one condition before moving on.
      strategy:
        type: string
        enum: [coverage, depth]
        deprecated: true
        description: "Deprecated alias of ordering (dual-read)."
      runner:
        type: string
        minLength: 1
        default: native
        description: >
          Lab adapter id (mas.lab.runners entry point). Optional override —
          when omitted, inferred from application.app / mas.manifest
          spec.framework.default_adapter or agent spec.framework_adapter.
      reset_state:
        type: boolean
        description: "Reset emulation/runtime state between runs when true."

  bench_emulation:
    type: object
    additionalProperties: false
    description: "Emulation profile (infra llm/tools mode, runtime cache policy)."
    properties:
      infra:
        type: object
        additionalProperties: true
      runtime:
        type: object
        additionalProperties: true
      intercept:
        type: object
        additionalProperties: true
      faults:
        type: object
        additionalProperties: true
      pipeline:
        type: object
        additionalProperties: false
        description: >
          Stream-first pipeline executor settings (artefact store and telemetry
          channel). Never a CLI flag.
        properties:
          engine:
            type: string
            description: Ignored. The stream-first engine is the only executor.
          shadow:
            type: boolean
            description: Ignored. Shadow mode was removed with the dual-engine period.
          store:
            type: string
            description: "Artefact store root (default: next to the trace cache)."
          max_active:
            type: integer
            minimum: 1
            description: "Global cap on in-flight pipeline nodes (default: 8)."
          streams:
            type: object
            additionalProperties: true
            properties:
              transport:
                type: string
                enum: [inproc, log, websocket]
                default: inproc
              max_active:
                type: integer
                minimum: 1
                description: "Override max_active under streams:."

  # ── Artifact declarations ─────────────────────────────────────────────────
  artifact_declarations:
    type: object
    description: >
      Named artifacts at this scope level.  Keys are artifact names.
      Short form: name: type (string).
      Long form: name: {type, path, validate}.
    additionalProperties:
      oneOf:
        - type: string
          description: "Short form — artifact type name (e.g. 'trace', 'metrics')."
        - type: object
          required: [type]
          additionalProperties: false
          properties:
            type:
              type: string
              description: >
                Artifact type from a library.yaml plugin (type: artifact).
            path:
              type: string
              description: >
                Output path template.  Supports variables: {run_dir}, {test_dir},
                {scenario_dir}, {scenario_id}, {item_id}, {run_idx}.
                Default is derived from the artifact type.
            validate:
              type: boolean
              default: false
              description: "Validate artifact schema after creation."

  # ── Level section (item / scenario / experiment-level hooks) ──────────
  level_section:
    type: object
    additionalProperties: false
    description: >-
      Pre/post hooks and artifacts at a single hierarchy level.
    properties:
      artifacts:
        $ref: "#/$defs/artifact_declarations"
      pre:
        type: array
        description: "Pipelines to run before this level's execution."
        items:
          $ref: "#/$defs/level_pipeline_entry"
      post:
        type: array
        description: "Pipelines to run after this level's execution."
        items:
          $ref: "#/$defs/level_pipeline_entry"

  # ── Run level section ────────────────────────────────────────────────────
  # Extends level_section with n_runs — the only per-run execution parameter
  # that belongs at this scope (not in execution:).
  run_level_section:
    type: object
    additionalProperties: false
    description: >-
      Pre/post hooks, artifacts, and run-count declaration for the run scope.
      n_runs is declared here rather than in execution: to keep the execution
      block free of per-run settings.
    properties:
      n_runs:
        type: integer
        minimum: 1
        default: 3
        description: "How many times each (scenario × item) combination is executed."
      artifacts:
        $ref: "#/$defs/artifact_declarations"
      pre:
        type: array
        description: "Pipelines to run before each run (0, 1, or N pipelines)."
        items:
          $ref: "#/$defs/level_pipeline_entry"
      post:
        type: array
        description: "Pipelines to run after each run."
        items:
          $ref: "#/$defs/level_pipeline_entry"

  # ── Level hook entry (inline step, file ref, or wrapped steps) ───────────
  level_pipeline_entry:
    description: >
      One pipeline step, external pipeline file path, or wrapped step list.
    oneOf:
      - $ref: "#/$defs/pipeline_step"
      - type: string
        description: "Path to an external pipeline.yaml (resolved relative to the experiment)."
      - type: object
        additionalProperties: false
        properties:
          steps:
            type: array
            items:
              $ref: "#/$defs/pipeline_step"
          ref:
            type: string
            description: "Path to an external pipeline.yaml file."
          id:
            type: string
            description: "Named pipeline ID from the pipeline library."
        oneOf:
          - required: [steps]
          - required: [ref]
          - required: [id]

  # ── Pipeline phase (inline or by reference) ───────────────────────────────
  pipeline_phase:
    description: >
      A pipeline attached to a level phase (pre or post).  Three equivalent forms:

        # Inline shorthand — list of steps directly:
        post:
          - {type: extract_trajectories}
          - {type: eval_mce, depends_on: [extract_trajectories]}

        # File reference — path to an external pipeline.yaml:
        post: ./pipelines/post-run.yaml

        # Long form — explicit steps or ref:
        post:
          steps:
            - {type: extract_trace_stats, depends_on: [extract_trajectories]}
        post:
          ref: ./pipelines/post-run.yaml
    oneOf:
      - type: string
        description: "Path to an external pipeline.yaml file (resolved relative to the experiment)."
      - type: array
        description: "Inline shorthand — list of steps directly."
        items:
          $ref: "#/$defs/pipeline_step"
      - type: object
        additionalProperties: false
        description: "Long form — either inline steps, an external file reference, or a library pipeline ID."
        properties:
          steps:
            type: array
            items:
              $ref: "#/$defs/pipeline_step"
          ref:
            type: string
            description: "Path to an external pipeline.yaml file."
          id:
            type: string
            description: "Named pipeline ID resolved from the pipeline library."
        oneOf:
          - required: [steps]
          - required: [ref]
          - required: [id]

  # ── Pipeline step ─────────────────────────────────────────────────────────
  pipeline_step:
    type: object
    required: [type]
    additionalProperties: false
    description: "One step in a pipeline."
    properties:
      type:
        type: string
        description: "Registered step type."
      name:
        type: string
        description: "Explicit step name (auto-generated when absent)."
      in:
        description: >
          Input artifact name(s).  Scalar (string) for a single input,
          array for multiple.  When a step at level L references an artifact
          declared at level L-1 (child), it fans in all instances.
        oneOf:
          - type: string
          - type: array
            items:
              type: string
      out:
        description: >
          Output artifact name(s).  Scalar or array.  Must match artifacts
          declared at this level.
        oneOf:
          - type: string
          - type: array
            items:
              type: string
      depends_on:
        type: array
        items:
          type: string
        description: "Names of upstream steps that must complete first."
      per_run:
        type: boolean
        default: false
        description: >
          When true, this step is materialized once per discovered benchmark
          run folder. Use with concurrency to control parallel execution.
      per_scenario:
        type: boolean
        default: false
        description: "When true, this step is materialized once per scenario."
      concurrency:
        type: integer
        minimum: 1
        default: 1
        description: >
          Maximum number of run-scope instances that may execute concurrently.
          Only effective when scope: run (or per_run: true).
          concurrency: 1 (default) preserves sequential behavior.
          concurrency: N > 1 wraps all run instances in a concurrent runner
          backed by asyncio.Semaphore(N).
      config:
        type: object
        additionalProperties: true
        description: "Step-type-specific configuration."