//! The statistical reductions against LibreOffice: every dataset of
//! `tests/data/statistical_reduce_goldens.tsv` (regenerated by
//! `scripts/goldens/statistical_reduce.py`) becomes a model with category T
//! (one item per row), optionally G (one item per group), and one input
//! measure per column. Each golden formula, e.g. `STDEV(X OVER T)`, is a
//! derived measure evaluated by the dataflow engine and compared, group by
//! group (and item by item for `RANK`-style reductions), with the value
//! LibreOffice computed over the same cells.

use improv_core_model::{
    CategoryId, Coordinate, ItemId, Measure, MeasureId, MeasureKind, Model, Name, Value,
    ValueError, ValueErrorKind, ValueType,
};
use improv_engine::{compiler::derived_shape, dataflow::evaluate, CellValue};
use serde_json::Value as J;

const T: CategoryId = CategoryId(1);
const G: CategoryId = CategoryId(2);

fn t_item(i: usize) -> ItemId {
    ItemId(100 + i as u32)
}
fn g_item(g: usize) -> ItemId {
    ItemId(200 + g as u32)
}

fn error_kind(j: &J) -> Option<ValueErrorKind> {
    let code = j.get("error")?.as_str()?;
    Some(ValueErrorKind::from_code(code).unwrap_or_else(|| panic!("unknown error {code}")))
}

fn input(j: &J) -> Option<Value> {
    Some(match j {
        J::Null => return None,
        J::Bool(b) => Value::Boolean(*b),
        J::Number(n) => Value::Number(n.as_f64().unwrap()),
        J::String(s) => Value::Text(s.clone()),
        other => Value::Error(ValueError::new(error_kind(other).expect("an error object"))),
    })
}

/// A golden cell: a number, an error object, `{"approx": x, "tol": t}`, or
/// null for no cell.
fn expected(j: &J) -> Option<CellValue> {
    match j {
        J::Null => None,
        J::Object(o) if o.contains_key("approx") => Some(CellValue::num(o["approx"].as_f64()?)),
        J::Number(n) => Some(CellValue::num(n.as_f64().unwrap())),
        other => Some(CellValue::Err(
            error_kind(other)
                .and_then(|k| k.error_type())
                .expect("error code"),
        )),
    }
}

/// The dataset's model: categories T (and G), one input measure per column.
fn model(data: &J) -> Model {
    let mut m = Model::new();
    let n_t = data["T"].as_u64().unwrap() as usize;
    let n_g = data.get("G").and_then(J::as_u64).map(|g| g as usize);
    m.add_category(T, "T");
    for i in 0..n_t {
        m.add_item(t_item(i), T, format!("t{i}"));
    }
    if let Some(n_g) = n_g {
        m.add_category(G, "G");
        for g in 0..n_g {
            m.add_item(g_item(g), G, format!("g{g}"));
        }
    }
    let measures = data["m"].as_object().unwrap();
    for (k, (name, groups)) in measures.iter().enumerate() {
        let id = MeasureId(10 + k as u32);
        let mut vt = ValueType::Number;
        for (g, vals) in groups.as_array().unwrap().iter().enumerate() {
            for (i, v) in vals.as_array().unwrap().iter().enumerate() {
                let Some(v) = input(v) else { continue };
                vt = match &v {
                    Value::Text(_) => ValueType::Text,
                    Value::Boolean(_) => ValueType::Boolean,
                    _ => vt,
                };
                let mut pairs = vec![(T, t_item(i))];
                if n_g.is_some() {
                    pairs.push((G, g_item(g)));
                }
                m.set_input(id, Coordinate::from_pairs(pairs), v);
            }
        }
        let mut cats = vec![T];
        if n_g.is_some() {
            cats.push(G);
        }
        m.add_measure(Measure {
            id,
            name: Name(name.clone()),
            value_type: vt,
            categories: cats,
            kind: MeasureKind::Input,
            description: None,
        });
    }
    m
}

/// Equal, or within 1e-12 relative (fits and tail probabilities may differ
/// from LibreOffice's in the last bits), or an `approx` row's `tol`.
fn close(got: &CellValue, want: &CellValue, j: &J) -> bool {
    let tol = j.get("tol").and_then(J::as_f64).unwrap_or(1e-12);
    match (got.as_num(), want.as_num()) {
        (Some(a), Some(b)) => a == b || (a - b).abs() <= tol * b.abs().max(1e-300),
        _ => got == want,
    }
}

/// A golden row: (line, formula, expected per group, source).
type Case = (usize, String, J, String);

#[test]
fn statistical_reductions_match_libreoffice() {
    let text = include_str!("data/statistical_reduce_goldens.tsv");
    let mut datasets: Vec<(String, J, Vec<Case>)> = Vec::new();
    for (n, line) in text.lines().enumerate().skip(1) {
        let cols: Vec<&str> = line.split('\t').collect();
        if let Some(name) = cols[0].strip_prefix('@') {
            datasets.push((
                name.into(),
                serde_json::from_str(cols[1]).unwrap(),
                Vec::new(),
            ));
            continue;
        }
        let ds = datasets.last_mut().expect("a dataset line first");
        assert_eq!(cols[0], ds.0, "line {}", n + 1);
        let want = serde_json::from_str(cols[2]).unwrap();
        ds.2.push((n + 1, cols[1].into(), want, cols[3].into()));
    }
    let mut failures = Vec::new();
    let mut checked = 0;
    for (name, data, cases) in &datasets {
        let mut m = model(data);
        let mut targets = Vec::new();
        for (k, (line, formula, want, src)) in cases.iter().enumerate() {
            let id = MeasureId(1000 + k as u32);
            let text = format!("R{k} = {formula}");
            let f = match improv_core_model::parse_formula(&m, &text) {
                Ok(f) => f.formula,
                Err(e) => {
                    failures.push(format!("{line}: {formula}: parse error {e}"));
                    continue;
                }
            };
            let (vt, cats) = match derived_shape(&m, id, &format!("R{k}"), &f) {
                Ok(s) => s,
                Err(e) => {
                    failures.push(format!("{line}: {formula}: compile error {e}"));
                    continue;
                }
            };
            m.add_measure(Measure {
                id,
                name: Name(format!("R{k}")),
                value_type: vt,
                categories: cats,
                kind: MeasureKind::Derived(f),
                description: None,
            });
            targets.push((id, *line, formula, want, src));
        }
        let ids: Vec<MeasureId> = targets.iter().map(|t| t.0).collect();
        let out = evaluate(&m, &ids).unwrap_or_else(|e| panic!("dataset {name}: {e}"));
        let grouped = data.get("G").is_some();
        for (id, line, formula, want, src) in targets {
            let res = out.get(&id).cloned().unwrap_or_default();
            for (g, w) in want.as_array().unwrap().iter().enumerate() {
                let gk: Vec<(u32, u32)> = if grouped {
                    vec![(G.0, g_item(g).0)]
                } else {
                    vec![]
                };
                let cells: Vec<(Vec<(u32, u32)>, &J)> = match w {
                    J::Array(items) => items
                        .iter()
                        .enumerate()
                        .map(|(i, w)| {
                            let mut k = gk.clone();
                            k.push((T.0, t_item(i).0));
                            k.sort_unstable();
                            (k, w)
                        })
                        .collect(),
                    w => vec![(gk.clone(), w)],
                };
                for (key, w) in cells {
                    let want = expected(w);
                    let got = res.get(&key);
                    let ok = match (got, &want) {
                        (Some(g), Some(want)) => close(g, want, w),
                        (None, None) => true,
                        _ => false,
                    };
                    checked += 1;
                    if !ok {
                        failures.push(format!(
                            "{line}: {formula} at {key:?} = {got:?}, want {want:?} ({src})"
                        ));
                    }
                }
            }
        }
    }
    assert!(checked > 400, "only {checked} cells checked");
    assert!(
        failures.is_empty(),
        "{} of {checked} cells fail:\n{}",
        failures.len(),
        failures.join("\n")
    );
}