//! The P7 analysis reductions against LibreOffice and Gnumeric: every
//! dataset of `tests/data/analysis_reduce_goldens.tsv` (regenerated by
//! `scripts/goldens/analysis_reduce.py`) becomes a model with its
//! categories (`cats`, in order; G, when present, is a group category every
//! measure keeps) and one input measure per entry of `m` (the categories it
//! is reduced over, then per group its values flattened row by row). Each
//! golden formula, e.g. `MINVERSE(A OVER R, C INTO P, Q)`, is a derived
//! measure evaluated by the dataflow engine; its results are compared cell
//! by cell: one per group for an aggregation, one per item of T for a
//! per-item function, and for an array one per cell of the INTO categories
//! (row-major, null for no cell).

use improv_core_model::{
    CategoryId, Coordinate, ItemId, Measure, MeasureId, MeasureKind, Model, Name, Value,
    ValueError, ValueErrorKind, ValueType,
};
use improv_engine::{compiler::derived_shape, dataflow::evaluate, CellValue};
use serde_json::Value as J;
use std::collections::HashMap;

fn item(c: CategoryId, i: usize) -> ItemId {
    ItemId(100 * c.0 + i as u32)
}

fn error_kind(j: &J) -> Option<ValueErrorKind> {
    let code = j.get("error")?.as_str()?;
    Some(ValueErrorKind::from_code(code).unwrap_or_else(|| panic!("unknown error {code}")))
}

fn input(j: &J) -> Value {
    match j {
        J::Bool(b) => Value::Boolean(*b),
        J::Number(n) => Value::Number(n.as_f64().unwrap()),
        J::String(s) => Value::Text(s.clone()),
        other => Value::Error(ValueError::new(error_kind(other).expect("an error object"))),
    }
}

/// A golden cell: a number, text, an error object, or null for no cell.
fn expected(j: &J) -> Option<CellValue> {
    Some(match j {
        J::Null => return None,
        J::Number(n) => CellValue::num(n.as_f64().unwrap()),
        J::String(s) => CellValue::Text(s.clone()),
        other => CellValue::Err(
            error_kind(other)
                .and_then(|k| k.error_type())
                .expect("error code"),
        ),
    })
}

/// The dataset's model and its categories by name.
fn model(data: &J) -> (Model, HashMap<String, (CategoryId, usize)>) {
    let mut m = Model::new();
    let mut cats = HashMap::new();
    for (k, (name, n)) in data["cats"].as_object().unwrap().iter().enumerate() {
        let c = CategoryId(1 + k as u32);
        let n = n.as_u64().unwrap() as usize;
        m.add_category(c, name);
        for i in 0..n {
            m.add_item(item(c, i), c, format!("{name}{i}"));
        }
        cats.insert(name.clone(), (c, n));
    }
    let g = cats.get("G").copied();
    for (k, (name, spec)) in data["m"].as_object().unwrap().iter().enumerate() {
        let id = MeasureId(10 + k as u32);
        let over: Vec<(CategoryId, usize)> = spec[0]
            .as_array()
            .unwrap()
            .iter()
            .map(|c| cats[c.as_str().unwrap()])
            .collect();
        let mut vt = ValueType::Number;
        for (gi, vals) in spec[1].as_array().unwrap().iter().enumerate() {
            for (i, v) in vals.as_array().unwrap().iter().enumerate() {
                let v = input(v);
                vt = match &v {
                    Value::Text(_) => ValueType::Text,
                    Value::Boolean(_) => ValueType::Boolean,
                    _ => vt,
                };
                // Row-major: the last category varies fastest.
                let (mut pairs, mut rest) = (Vec::new(), i);
                for (c, n) in over.iter().rev() {
                    pairs.push((*c, item(*c, rest % n)));
                    rest /= n;
                }
                if let Some((gc, _)) = g {
                    pairs.push((gc, item(gc, gi)));
                }
                m.set_input(id, Coordinate::from_pairs(pairs), v);
            }
        }
        let mut categories: Vec<CategoryId> = over.iter().map(|(c, _)| *c).collect();
        categories.extend(g.map(|(c, _)| c));
        m.add_measure(Measure {
            id,
            name: Name(name.clone()),
            value_type: vt,
            categories,
            kind: MeasureKind::Input,
            description: None,
        });
    }
    (m, cats)
}

/// Equal, or within 1e-12 relative to max(|want|, 1): the results come
/// from different algorithms (Jacobi SVD here, quad-precision QR in
/// Gnumeric), which agree to about ten ulps.
fn close(got: &CellValue, want: &CellValue) -> bool {
    let near = |a: f64, b: f64| a == b || (a - b).abs() <= 1e-12 * b.abs().max(1.0);
    match (got.as_num(), want.as_num(), got, want) {
        (Some(a), Some(b), ..) => near(a, b),
        // Complex text (FOURIER): each part within the tolerance, since
        // sin/cos differ by an ulp between platforms' libms.
        (_, _, CellValue::Text(g), CellValue::Text(w)) => match (complex(g), complex(w)) {
            (Some((gr, gi)), Some((wr, wi))) => near(gr, wr) && near(gi, wi),
            _ => g == w,
        },
        _ => got == want,
    }
}

/// `"a+bi"` / `"a-bi"` / `"a"` / `"bi"` as (re, im).
fn complex(s: &str) -> Option<(f64, f64)> {
    let Some(body) = s.strip_suffix(['i', 'j']) else {
        return s.parse().ok().map(|r| (r, 0.0));
    };
    // The sign that starts the imaginary part: the last + or - not at the
    // start and not after an exponent's e.
    let b = body.as_bytes();
    let at = (1..b.len())
        .rev()
        .find(|&k| matches!(b[k], b'+' | b'-') && !matches!(b[k - 1], b'e' | b'E'));
    let (re, im) = match at {
        Some(k) => (body[..k].parse().ok()?, &body[k..]),
        None => (0.0, body),
    };
    let im = match im {
        "" | "+" => 1.0,
        "-" => -1.0,
        x => x.parse().ok()?,
    };
    Some((re, im))
}

#[test]
fn complex_text_parses() {
    assert_eq!(complex("-0.625+1.8i"), Some((-0.625, 1.8)));
    assert_eq!(complex("1e-3-2e-3i"), Some((1e-3, -2e-3)));
    assert_eq!(complex("-i"), Some((0.0, -1.0)));
    assert_eq!(complex("2"), Some((2.0, 0.0)));
    assert_eq!(complex("x"), None);
    assert!(close(
        &CellValue::Text("-0.625+1.8106601717798214i".into()),
        &CellValue::Text("-0.6250000000000001+1.8106601717798214i".into())
    ));
    assert!(!close(
        &CellValue::Text("1+i".into()),
        &CellValue::Text("1+2i".into())
    ));
}

/// The result cells' keys in golden order: the cells of `list` (row-major)
/// within group `g`.
fn keys(
    list: &[(CategoryId, usize)],
    g: Option<(CategoryId, usize)>,
    gi: usize,
) -> Vec<Vec<(u32, u32)>> {
    let mut out: Vec<Vec<(u32, u32)>> = vec![Vec::new()];
    for (c, n) in list {
        out = out
            .into_iter()
            .flat_map(|pre| {
                (0..*n).map(move |i| {
                    let mut k = pre.clone();
                    k.push((c.0, item(*c, i).0));
                    k
                })
            })
            .collect();
    }
    for k in &mut out {
        k.extend(g.map(|(c, _)| (c.0, item(c, gi).0)));
        k.sort_unstable();
    }
    out
}

/// A golden row: (line, formula, expected per group, source).
type Case = (usize, String, J, String);

#[test]
fn analysis_functions_match_libreoffice_and_gnumeric() {
    let text = include_str!("data/analysis_reduce_goldens.tsv");
    let mut datasets: Vec<(String, J, Vec<Case>)> = Vec::new();
    for (n, line) in text.lines().enumerate().skip(1) {
        let cols: Vec<&str> = line.split('\t').collect();
        if let Some(name) = cols[0].strip_prefix('@') {
            datasets.push((
                name.into(),
                serde_json::from_str(cols[1]).unwrap(),
                Vec::new(),
            ));
            continue;
        }
        let ds = datasets.last_mut().expect("a dataset line first");
        assert_eq!(cols[0], ds.0, "line {}", n + 1);
        let want = serde_json::from_str(cols[2]).unwrap();
        ds.2.push((n + 1, cols[1].into(), want, cols[3].into()));
    }
    let (mut failures, mut checked) = (Vec::new(), 0);
    let mut sources: HashMap<String, usize> = HashMap::new();
    for (name, data, cases) in &datasets {
        let (mut m, cats) = model(data);
        let g = cats.get("G").copied();
        let mut targets = Vec::new();
        for (k, (line, formula, want, src)) in cases.iter().enumerate() {
            let id = MeasureId(1000 + k as u32);
            let text = format!("R{k} = {formula}");
            let f = match improv_core_model::parse_formula(&m, &text) {
                Ok(f) => f.formula,
                Err(e) => {
                    failures.push(format!("{line}: {formula}: parse error {e}"));
                    continue;
                }
            };
            let (vt, rcats) = match derived_shape(&m, id, &format!("R{k}"), &f) {
                Ok(s) => s,
                Err(e) => {
                    failures.push(format!("{line}: {formula}: compile error {e}"));
                    continue;
                }
            };
            // The listed categories: INTO's in order, else the result's
            // (one per item of T for a per-item function) but G.
            let list: Vec<(CategoryId, usize)> = match formula.rsplit_once(" INTO ") {
                Some((_, into)) => into
                    .trim_end_matches(')')
                    .split(", ")
                    .map(|c| cats[c])
                    .collect(),
                None => rcats
                    .iter()
                    .filter(|c| Some(**c) != g.map(|g| g.0))
                    .map(|c| *cats.values().find(|(cc, _)| cc == c).unwrap())
                    .collect(),
            };
            m.add_measure(Measure {
                id,
                name: Name(format!("R{k}")),
                value_type: vt,
                categories: rcats,
                kind: MeasureKind::Derived(f),
                description: None,
            });
            targets.push((id, *line, formula, want, src, list));
        }
        let ids: Vec<MeasureId> = targets.iter().map(|t| t.0).collect();
        let out = evaluate(&m, &ids).unwrap_or_else(|e| panic!("dataset {name}: {e}"));
        for (id, line, formula, want, src, list) in targets {
            let res = out.get(&id).cloned().unwrap_or_default();
            let mut seen = 0;
            for (gi, cells) in want.as_array().unwrap().iter().enumerate() {
                let cells = cells.as_array().unwrap();
                let ks = keys(&list, g, gi);
                assert_eq!(ks.len(), cells.len(), "{line}: {formula}: golden shape");
                for (key, w) in ks.iter().zip(cells) {
                    let want = expected(w);
                    let got = res.get(key);
                    seen += usize::from(got.is_some());
                    let ok = match (got, &want) {
                        (Some(g), Some(want)) => close(g, want),
                        (None, None) => true,
                        _ => false,
                    };
                    checked += 1;
                    if !ok {
                        failures.push(format!(
                            "{line}: {formula} at {key:?} = {got:?}, want {want:?} ({src})"
                        ));
                    }
                }
            }
            *sources.entry(src.clone()).or_default() += 1;
            if res.len() != seen {
                failures.push(format!(
                    "{line}: {formula}: {} cells, want {seen}",
                    res.len()
                ));
            }
        }
    }
    assert!(checked > 100, "only {checked} cells checked");
    assert!(
        sources.contains_key("LibreOffice") && sources.contains_key("Gnumeric 1.12"),
        "both products are sources: {sources:?}"
    );
    assert!(
        failures.is_empty(),
        "{} of {checked} cells fail:\n{}",
        failures.len(),
        failures.join("\n")
    );
}