# Mid-hardness synthetic 1M corpus (recalibrated
# mid-hardness synthetic). Same full filter/concurrency matrix as
# full_1m.yaml; ONLY the dataset geometry differs. This corpus carries the
# filtered headline bars; the original
# hard corpus (full_1m.yaml, 1000 centers ~ ivfflat's lists heuristic,
# unfiltered hnsw ceiling 0.677) survives as a labeled stress appendix only.
#
# CALIBRATION EVIDENCE (all measured by THIS harness, hnsw m=16
# ef_construction=64, k=10, l2, seed 42; recall = unfiltered recall@10 at 1
# conn; runs of 2026-09-10/11 on this machine, raw JSONs in results/ as
# calib_*):
#
# Target: pgvector hnsw reaches ~0.90-0.95 unfiltered at moderate ef
# (>=0.90 at ef<=160), center count NOT near 1000 (breaks the structural
# ivfflat flattery of centers ~= lists).
#
# 1) Small-scale proxies DO NOT transfer to 1M -- measured, not assumed:
#    | n    | centers | sigma | scale_mix       | recall@ef160 |
#    | 100k |    600  | 1.1   | [0.6,1.0,1.6]   | 0.884 |
#    | 100k |    700  | 1.1   | [0.6,1.0,1.6]   | 0.922 |
#    | 200k |   1400  | 1.1   | [0.6,1.0,1.6]   | 0.857 |  (same density as 700@100k)
#    | 200k |   1400  | 0.85  | [0.6,1.0,1.6]   | 0.948 |
#    | 400k |   2800  | 0.85  | [0.6,1.0,1.6]   | 0.892 |
#    | 400k |   2800  | 0.7   | [0.6,1.0,1.6]   | 0.945 |
#    | 1M   |   7000  | 0.7   | [0.6,1.0,1.6]   | 0.817 |  <- the 400k 0.945 predicted this; off by 0.13
#    The task's original "verify on a 100k subsample" protocol was therefore
#    abandoned for direct 1M-scale iteration:
#    recall at fixed density degrades with scale, so only 1M measurements
#    count.
#
# 2) 1M-scale iteration trace (the sigma sweet spot is ~0.3-0.35 at 8000
#    centers; BOTH much tighter and much wider lose recall -- nonmonotonic):
#    | centers | sigma | scale_mix       | recall@ef160 |
#    |  7000   | 0.7   | [0.6,1.0,1.6]   | 0.817 |
#    |  8000   | 0.5   | [0.6,1.0,1.6]   | 0.858 |
#    |  8000   | 0.35  | [0.6,1.0,1.6]   | 0.892 |
#    |  8000   | 0.3   | [0.6,1.0,1.6]   | 0.890 |
#    |  8000   | 0.2   | [0.6,1.0,1.6]   | 0.803 |  <- too-tight collapse
#    |  9000   | 0.3   | [0.6,1.0,1.6]   | 0.858 |
#    |  8000   | 0.35  | [0.8,1.0,1.25]  | 0.904 @ ef80, 0.964 @ ef160  <- THIS CONFIG
#    The winning change: narrow cluster_scale_mix. With the old [0.6,1.0,1.6]
#    mix at base sigma 0.35, the 0.6x clusters sit at effective sigma 0.21 --
#    inside the too-tight regime that measurably collapses hnsw recall (see
#    the sigma-0.2 row). Removing that tail lifts ef160 recall 0.892 -> 0.964
#    and puts ef80 at 0.904: the 0.90-0.95 band is crossed between ef 80 and
#    160, i.e. reachable at moderate ef, which is the definition of
#    mid-hardness this corpus needed. Cheapest-point recall 0.544 (hardness
#    gate 0.97 passes with margin; the knob still matters: 0.54 -> 0.96
#    across the sweep). Raw evidence: results/calib_1m_c1_20260911T185124Z.*
#    plus the calib_midhardness_* series (2026-09-10, pruned corpora, JSONs
#    kept).
#
# 8000 centers is 8x ivfflat's lists heuristic at 1M (rows/1000 = 1000), so
# the near-circular centers~=lists flattery of the original corpus is gone.
name: full_1m_mid
seed: 42
k: 10
warmup_queries: 20
hardness_recall_threshold: 0.97
min_throughput_seconds: 5.0

postgres:
  maintenance_work_mem: 8GB
  min_wal_size: 1GB
  max_wal_size: 4GB

dataset:
  kind: synthetic
  n: 1000000
  dim: 768
  n_centers: 8000
  cluster_sigma: 0.35
  cluster_scale_mix: [0.8, 1.0, 1.25]
  background_frac: 0.02
  n_queries: 500

metric: l2

filters:
  selectivities: [0.001, 0.01, 0.10, 0.50]
  modes: [correlated, uncorrelated]
  run:
    - {selectivity: 0.001, mode: correlated}
    - {selectivity: 0.001, mode: uncorrelated}
    - {selectivity: 0.01, mode: correlated}
    - {selectivity: 0.01, mode: uncorrelated}
    - {selectivity: 0.10, mode: correlated}
    - {selectivity: 0.10, mode: uncorrelated}
    - {selectivity: 0.50, mode: correlated}
    - {selectivity: 0.50, mode: uncorrelated}

systems:
  - name: pgvector_hnsw
    type: hnsw
    build_params: {m: 16, ef_construction: 64}
    sweep_param: ef_search
    sweep_values: [10, 40, 80, 160]
    iterative_scan_values: [null]
    filtered_iterative_scan_values: ["off", "relaxed_order"]
    scan_budget_param: hnsw.max_scan_tuples
    filtered_scan_budget_values: [20000, auto]
  - name: pgvector_ivfflat
    type: ivfflat
    build_params: {lists: null}
    sweep_param: probes
    sweep_values: [1, 2, 4, 8, 16]
    iterative_scan_values: [null]
    filtered_iterative_scan_values: ["off", "relaxed_order"]

concurrency: [1, 8]

churn:
  enabled: true
  cycles: 3
  reembed_pct: 0.02
  delete_reinsert_pct: 0.02
  vacuum_after_each_cycle: true
  system: {name: pgvector_hnsw, type: hnsw, build_params: {m: 16, ef_construction: 64},
           sweep_param: ef_search, sweep_values: [80]}
  sweep_value: 80

# SIFT1M sanity deliberately OFF here: it is corpus-independent and already
# anchored by the full_1m run of record -- rerunning it on this config would
# only duplicate 1M x 128d build+sweep hours.
sift1m:
  enabled: false
