id area query/feature status condition engine example notes proof check query since avoid readme #feature aggs-order feat/v3-plan2-aggs test/col/fixtures/g_order_by.sql #feature aggs-distinct feat/v3-plan2-aggs test/col/fixtures/g_distinct_multi.sql #feature aggs-hypothetical feat/v3-plan2-aggs test/col/fixtures/g_hypothetical.sql #feature aggs-variadic feat/v3-plan2-aggs test/col/fixtures/g_variadic.sql #feature aggs-joined feat/v3-plan2-aggs test/col/fixtures/g_joined_column.sql #feature aggs-parallel feat/v3-plan2-aggs test/col/fixtures/g_parallel.sql #feature expr-timestamp feat/v3-plan2-expr test/col/fixtures/e_timestamp_expr.sql #feature expr-int feat/v3-plan2-expr test/col/fixtures/e_int8_expr.sql #feature expr-numeric feat/v3-plan2-expr test/col/fixtures/n_numeric_pg_filters.sql #feature arms-outer feat/v3-plan2-arms test/col/fixtures/per_order_hidden_sets.sql #feature arms-cells feat/v3-plan2-arms test/col/fixtures/per_order_global_distinct.sql #feature preagg feat/v3-preagg2 test/sql/agg_preagg.sql A01 Aggregate functions count(*) Yes sift_col, sift_table SELECT status, count(*) FROM orders GROUP BY status Global (no GROUP BY) too. fixture:f_count_sum_minmax,fixture:f_group_source agg SELECT status, count(*) FROM orders GROUP BY status 01:`count(*)`, `count(DISTINCT x)` A02 Aggregate functions count(col) Yes sift_col, sift_table SELECT country_code, count(country_code) FROM orders GROUP BY 1 NULLs are skipped as PostgreSQL does. fixture:n_null_columns agg SELECT country_code, count(country_code), count(customer_id) FROM orders GROUP BY 1 01:`count(*)`, `count(DISTINCT x)` A03 Aggregate functions count(DISTINCT col) Yes sift_col, sift_table SELECT status, count(DISTINCT tenant_id) FROM orders GROUP BY 1 Dictionary, integer, date and numeric(p<=18) columns, or an expression of them. fixture:f_distinct_dict,fixture:n_numeric_distinct agg SELECT status, count(DISTINCT tenant_id), count(DISTINCT total), count(DISTINCT lower(source)) FROM orders GROUP BY 1 01:`count(*)`, `count(DISTINCT x)` A04 Aggregate functions count(DISTINCT o.id) over an unnested array Yes sift_col, sift_table SELECT count(DISTINCT o.id) FROM orders o, jsonb_to_recordset(o.items) i(qty int) Distinct parent rows, proven by a unique NOT NULL column. fixture:f_distinct_parent agg @f_distinct_parent 01:`count(*)`, `count(DISTINCT x)` A05 Aggregate functions sum, avg of int2, int4, int8 Yes sift_col, sift_table SELECT grp, sum(i4), avg(i4) FROM t GROUP BY grp sum(int4) is bigint, avg is numeric, as PostgreSQL gives them. fixture:t_sum_int4_wide,fixture:t_sum_int8_numeric agg SELECT grp, sum(i2), sum(i4), avg(i4), sum(i8), avg(i8) FROM cv_types GROUP BY grp 02:`sum`, `avg`, `min`, `max` A06 Aggregate functions sum, avg of float4, float8 Yes sift_col, sift_table SELECT grp, sum(f8), avg(f4) FROM t GROUP BY grp Float sums add in a fixed lane order (same result at any worker count and SIMD level); the last digits can differ from the heap-order sum of the normal plan (relative 1e-9). fixture:t_float8_sum_many,fixture:t_float4_sum agg SELECT grp, sum(f8), avg(f8), sum(f4), avg(f4) FROM cv_types GROUP BY grp 02:`sum`, `avg`, `min`, `max` A07 Aggregate functions sum, avg, min, max of numeric(p<=18, s) Yes sift_col SELECT source, sum(total), avg(total) FROM orders GROUP BY 1 Exact in 128 bits; the scale of the result is PostgreSQL's. Expressions of such columns (+ - *, round, floor) are exact too. fixture:n_numeric_sums,test:agg_numeric agg SELECT source, sum(total), avg(total), min(total), max(total), sum(total - discount) FROM orders GROUP BY 1 02:`sum`, `avg`, `min`, `max` A08 Aggregate functions min, max of integers, dates, timestamps Yes sift_col, sift_table SELECT grp, min(d), max(tstz) FROM t GROUP BY grp fixture:f_count_sum_minmax agg SELECT grp, min(d), max(d), min(ts), max(tstz), min(i8) FROM cv_types GROUP BY grp 02:`sum`, `avg`, `min`, `max` A09 Aggregate functions min, max of text, varchar Yes sift_col, sift_table SELECT grp, min(tx), max(vc) FROM t GROUP BY grp Any collation: PostgreSQL's own comparison over the decoded text. fixture:t_text_collation agg SELECT grp, min(tx), max(vc), min(tx COLLATE "C") FROM cv_types GROUP BY grp 02:`sum`, `avg`, `min`, `max` A10 Aggregate functions stddev, variance family Yes sift_col, sift_table SELECT source, stddev(total), var_pop(tenant_id) FROM orders GROUP BY 1 Generic aggregate: PostgreSQL's own transition and final functions run over the scan. fixture:g_stats_numeric,fixture:g_stats_float agg @g_stats_float 03:`stddev`, `percentile_*`, `string_agg`, `FILTER (WHERE ...)` A11 Aggregate functions corr, covar_*, regr_* Yes sift_col, sift_table SELECT source, corr(total, tenant_id), regr_slope(total, tenant_id) FROM orders GROUP BY 1 Generic aggregate. fixture:g_stats_float agg SELECT source, corr(total, tenant_id), covar_pop(total, tenant_id), regr_slope(total, tenant_id) FROM orders GROUP BY 1 03:`stddev`, `percentile_*`, `string_agg`, `FILTER (WHERE ...)` A12 Aggregate functions bool_and, bool_or, every, bit_and, bit_or, bit_xor Yes sift_col SELECT source, bool_and(deleted_at IS NULL), bit_or(tenant_id) FROM orders GROUP BY 1 Generic aggregates. fixture:g_stats_numeric,test:aggstub_generic agg SELECT source, bool_and(deleted_at IS NULL), bool_or(total > 400), every(tenant_id > 0), bit_and(tenant_id), bit_or(tenant_id), bit_xor(tenant_id) FROM orders GROUP BY 1 03:`stddev`, `percentile_*`, `string_agg`, `FILTER (WHERE ...)` A13 Aggregate functions string_agg, array_agg, json_agg, jsonb_agg, json_object_agg Yes sift_col, sift_table SELECT source, array_agg(tenant_id) FROM orders GROUP BY 1 Generic aggregates take the rows in the order of the scan (the heap order of a sequential scan); the SQL result of these is unordered either way. fixture:g_array_agg agg @g_array_agg 03:`stddev`, `percentile_*`, `string_agg`, `FILTER (WHERE ...)` A14 Aggregate functions string_agg(x, ',' ORDER BY y), array_agg(x ORDER BY y) Yes sift_col, sift_table SELECT source, string_agg(status, ',' ORDER BY id) FROM orders GROUP BY 1 A tuplesort per group on the order keys, then the transition function. fixture:g_order_by agg SELECT source, string_agg(status, ',' ORDER BY id) FROM orders GROUP BY 1 aggs-order 03:`stddev`, `percentile_*`, `string_agg`, `FILTER (WHERE ...)` A15 Aggregate functions percentile_cont, percentile_disc with a constant fraction Yes sift_col, sift_table SELECT source, percentile_cont(0.5) WITHIN GROUP (ORDER BY total::float8) FROM orders GROUP BY 1 Native kernel: exact histogram of the values, spilled to a tuplesort beyond work_mem. Parallel. fixture:o_percentile_rows,fixture:o_percentile_spill agg SELECT source, percentile_cont(0.5) WITHIN GROUP (ORDER BY total::float8), percentile_disc(0.9) WITHIN GROUP (ORDER BY placed_at) FROM orders GROUP BY 1 03:`stddev`, `percentile_*`, `string_agg`, `FILTER (WHERE ...)` A16 Aggregate functions percentile_* with an array of fractions, over intervals, text or dates; mode() Yes sift_col, sift_table SELECT tenant_id % 5 AS g, mode() WITHIN GROUP (ORDER BY country_code) FROM orders GROUP BY 1 Generic ordered-set aggregates: one tuplesort per group. fixture:g_ordered_set agg @g_ordered_set 03:`stddev`, `percentile_*`, `string_agg`, `FILTER (WHERE ...)` A17 Aggregate functions User-defined aggregates (CREATE AGGREGATE) Partly transition and final functions not volatile sift_col, sift_table SELECT source, coldiff.sum_sq(total::float8) FROM orders GROUP BY 1 Any immutable or stable aggregate runs as a generic one, on the argument leaves. fixture:g_user_aggregate agg @g_user_aggregate A18 Aggregate functions Aggregate with a volatile transition function or argument No not a function of the data, evaluated per row and per group sift_col, sift_table SELECT source, coldiff.sum_vol(total::float8) FROM orders GROUP BY 1 Same result, normal plan. fixture:g_trap_volatile,fixture:t_volatile_agg_arg plain @g_trap_volatile 13:`RIGHT JOIN`, `SELECT DISTINCT`, volatile functions A19 Aggregate functions FILTER (WHERE ...) on any aggregate Yes sift_col, sift_table SELECT count(*) FILTER (WHERE status = 'paid'), sum(total) FILTER (WHERE total > 100) FROM orders fixture:f_filter_clause agg @f_filter_clause 03:`stddev`, `percentile_*`, `string_agg`, `FILTER (WHERE ...)` A20 Aggregate functions DISTINCT in an aggregate with one varying argument Yes sift_col, sift_table SELECT source, string_agg(DISTINCT status, ',') FROM orders GROUP BY 1 sum, avg, min, max, count, string_agg, array_agg, jsonb_agg (DISTINCT); fixture:g_distinct agg @g_distinct 01:`count(*)`, `count(DISTINCT x)` A21 Aggregate functions DISTINCT over several arguments, e.g. corr(DISTINCT a, b) Yes sift_col, sift_table SELECT source, corr(DISTINCT total, tenant_id) FROM orders GROUP BY 1 A tuplesort of the argument tuples. fixture:g_distinct_multi agg SELECT source, corr(DISTINCT total, tenant_id) FROM orders GROUP BY 1 aggs-distinct A22 Aggregate functions Hypothetical-set aggregates: rank, dense_rank, percent_rank, cume_dist WITHIN GROUP Yes sift_col, sift_table SELECT source, rank('paid') WITHIN GROUP (ORDER BY status) FROM orders GROUP BY 1 fixture:g_hypothetical agg SELECT source, rank('paid') WITHIN GROUP (ORDER BY status), dense_rank('paid') WITHIN GROUP (ORDER BY status) FROM orders GROUP BY 1 aggs-hypothetical 03:`stddev`, `percentile_*`, `string_agg`, `FILTER (WHERE ...)` A23 Aggregate functions Variadic aggregates and aggregates of more than four arguments Yes sift_col, sift_table SELECT source, cv_sum_all(tenant_id, id % 5, 3) FROM orders GROUP BY 1 fixture:g_variadic agg SELECT source, cv_sum_all(tenant_id, id % 5, 3), cv_sum_all(VARIADIC ARRAY[tenant_id, 1]) FROM orders GROUP BY 1 aggs-variadic A24 Aggregate functions Aggregates of the columns of a lookup-joined table: count, min, max, sum, avg Yes sift_col, sift_table SELECT o.status, sum(v.weight) FROM orders o LEFT JOIN map_vals v ON v.category = o.source GROUP BY 1 Taken per matching row; int, interval and float4 sums run PostgreSQL's own functions on the distinct values. fixture:t_mapped_agg_left,fixture:t_mapped_agg_pg agg @t_mapped_agg_pg 02:`sum`, `avg`, `min`, `max` A25 Aggregate functions Generic aggregate (stddev, string_agg(DISTINCT), ...) over a lookup-joined column Yes sift_col, sift_table SELECT s.kind, stddev(s.id) FROM orders o JOIN tenants s ON s.id = o.tenant_id GROUP BY 1 fixture:g_joined_column agg SELECT s.kind, stddev(s.id), array_agg(DISTINCT s.country_code ORDER BY s.country_code) FROM orders o JOIN tenants s ON s.id = o.tenant_id GROUP BY 1 aggs-joined A26 Aggregate functions Aggregate of an expression of leaves: sum(qty * (price - discount)) Yes sift_col, sift_table SELECT sum(i.qty * (i.price - i.discount)) FROM orders o, jsonb_to_recordset(o.items) i(qty int, price float8, discount float8) Up to four leaves per expression, evaluated per element. fixture:g_expr_args agg @g_expr_args 02:`sum`, `avg`, `min`, `max` A27 Aggregate functions Aggregate over a column that is not in the index No the scan reads only indexed columns sift_col SELECT status, count(memo) FROM orders GROUP BY 1 Put the column in the index to push it down. script:coverage plain SELECT status, count(memo), max(length(memo)) FROM orders GROUP BY 1 A28 Aggregate functions Aggregates in ROLLUP, CUBE, GROUPING SETS Yes sift_col, sift_table SELECT tenant_id, status, count(*) FROM orders GROUP BY ROLLUP (tenant_id, status) All sets in one scan; GROUPING() and the NULLs of the sets are PostgreSQL's. fixture:t_sets_cube_having,fixture:g_rollup agg @t_sets_cube_having 05:`GROUP BY x`, `GROUP BY expr`, `ROLLUP`, `CUBE` A29 Aggregate functions Aggregate over a numeric expression PostgreSQL evaluates: division, %, power, sqrt, products past 38 digits, casts, NaN Yes sift_col, sift_table SELECT grp, sum(n2 / 3), max(power(n2, 2)) FROM t GROUP BY grp A generic aggregate on the argument expression (up to four leaves), evaluated by PostgreSQL per row; the numeric arithmetic is PostgreSQL's own. A kernel aggregate (sum, avg of + - * round) stays exact in 128 bits. fixture:n_numeric_pg_arith,fixture:n_numeric_pg_funcs,fixture:n_numeric_pg_special,fixture:n_numeric_pg_nan agg SELECT grp, sum(n2 / 3), max(n2 * n2 * n2 * n2), max(power(n2, 2)), max(n2::numeric(8, 1)) FROM cv_types GROUP BY grp 02:`sum`, `avg`, `min`, `max` A30 Aggregate functions min, max, sum, avg of DISTINCT numeric values, percentile_disc and mode() of a numeric column Yes sift_col, sift_table SELECT grp, min(DISTINCT n2), percentile_disc(0.5) WITHIN GROUP (ORDER BY n2) FROM t GROUP BY grp A generic aggregate over the decoded numeric values; the distinct values and the ordered set go through PostgreSQL's transition and final functions. fixture:n_numeric_trap_minmax_distinct,fixture:n_numeric_trap_percentile agg SELECT grp, min(DISTINCT n2), max(DISTINCT n2), percentile_disc(0.5) WITHIN GROUP (ORDER BY n2) FROM cv_types GROUP BY grp 02:`sum`, `avg`, `min`, `max` A31 Aggregate functions percentile_cont, percentile_disc with a fraction that is a stable expression or a parameter Yes sift_col, sift_table SELECT source, percentile_cont(least(1.0, extract(month FROM now())::float8 / 12.0)) WITHIN GROUP (ORDER BY total::float8) FROM orders GROUP BY 1 The fraction is evaluated when the scan starts. A fraction that is a column of the group keeps the normal plan (PostgreSQL evaluates it per group and raises its own error outside [0, 1]). fixture:g_ordered_set,fixture:o_percentile_array agg SELECT source, percentile_cont(least(1.0, extract(month FROM now())::float8 / 12.0)) WITHIN GROUP (ORDER BY total::float8), percentile_disc(ARRAY[0.25, 0.75]) WITHIN GROUP (ORDER BY placed_at) FROM orders GROUP BY source 03:`stddev`, `percentile_*`, `string_agg`, `FILTER (WHERE ...)` A32 Aggregate functions sift.approx_count_distinct(col): a HyperLogLog estimate of the distinct values of a column, from the sketches stored with the units (hll_columns) Yes ungrouped; units that are not exact (a dead row, a cut predicate) hash their rows; count(DISTINCT) stays exact in every plan sift_col CREATE INDEX ON t USING sift_col (k, u) WITH (hll_columns = 'k, u'); SELECT sift.approx_count_distinct(k) FROM t Opt-in (column format minor 20; a 3.3 library cannot read it). The estimate is the same number in every plan, with workers and by partition (relative standard error 0.81%); an index without the option answers it by hashing the rows. test:col_approx agg -- pre: DROP TABLE IF EXISTS cov_hll;\n-- pre: CREATE TABLE cov_hll (id int, g int, k int, u uuid) WITH (autovacuum_enabled = off);\n-- pre: INSERT INTO cov_hll SELECT i, i % 7, (i * 7919) % 20000, md5(i::text)::uuid FROM generate_series(1, 60000) i;\n-- pre: CREATE INDEX cov_hll_col ON cov_hll USING sift_col (id, g, k, u) WITH (hll_columns = 'k, u');\n-- pre: VACUUM (FREEZE, ANALYZE) cov_hll;\nSELECT sift.approx_count_distinct(k), sift.approx_count_distinct(u) FROM cov_hll AP01 Pre-aggregates count, sum, avg, min, max of integer, date and numeric columns without GROUP BY Yes sift_col SELECT count(*), sum(total), avg(discount), min(placed_at) FROM orders A unit whose rows are all visible and alive is answered from the sums and zone the index stored with it (`Units From Pre-aggregates` in EXPLAIN ANALYZE), the other units are scanned in the same plan; a comparison on an integer or date column that a unit's zone satisfies as a whole keeps the unit answerable. Float columns, count(DISTINCT), expressions and other tests scan. `sift.agg_preagg = off` scans every unit. fixture:pa_whole,fixture:pa_zone_global,test:agg_preagg agg SELECT count(*), count(status), sum(total), avg(discount), min(tenant_id), max(placed_at) FROM orders WHERE id <= 2000 09:pre-aggregated `count`, `sum`, `avg`, `min`, `max` AP02 Pre-aggregates GROUP BY one dictionary, date or small-integer column: count(*), count(col), sum and avg of numeric or integer columns Yes sift_col SELECT source, count(*), sum(total) FROM orders GROUP BY source Answered from the per-unit histograms of the column (`WITH (preagg = 'cube(source: total, discount)')` chooses the columns and measures; the default keeps up to 256 groups and 3 measures per unit within 2% of its size). A unit without the histogram (more groups, a measure it lacks) is scanned. fixture:pa_group_dim,fixture:pa_group_null,fixture:pa_group_small,fixture:pa_having_limit,test:agg_preagg agg SELECT status, count(*), count(status), sum(discount), avg(total) FROM orders GROUP BY status 09:pre-aggregated `count`, `sum`, `avg`, `min`, `max` AP03 Pre-aggregates The same with a test on the grouped column: =, IN, IS NULL, LIKE, comparisons Yes sift_col SELECT status, count(*), sum(total) FROM orders WHERE status IN ('paid', 'shipped') GROUP BY status The test runs on the groups of the histogram, as it runs on the rows of a unit. A global aggregate of the rows of one value works the same way. fixture:pa_dim_in,fixture:pa_dim_eq,fixture:pa_dim_null,fixture:pa_dim_like,test:agg_preagg agg SELECT count(*), sum(total) FROM orders WHERE source = 'web' 09:pre-aggregated `count`, `sum`, `avg`, `min`, `max` AP04 Pre-aggregates The same with a comparison on another integer or date column Yes sift_col SELECT source, count(*), sum(total) FROM orders WHERE placed_at >= DATE '2024-01-01' GROUP BY source A unit answers when its zone of that column lies inside the comparison and holds no NULL (time-ordered data: all but the units at the edge of the range); the others are scanned. A test on a column that is neither the grouped one nor such a comparison scans every unit. fixture:pa_zone_all,fixture:pa_zone_some,test:agg_preagg agg SELECT source, count(*), sum(total) FROM orders WHERE id > 0 AND placed_at >= DATE '2000-01-01' GROUP BY source 09:pre-aggregated `count`, `sum`, `avg`, `min`, `max` T01 Types int2, int4, int8 (columns, keys, filters, aggregates) Yes sift_col, sift_table SELECT i2, count(*), sum(i8) FROM t GROUP BY i2 fixture:t_sum_int4_wide,fixture:n_numeric_key_int8 agg SELECT i2, i4 % 3, count(*), sum(i8), avg(i4) FROM cv_types GROUP BY 1, 2 T02 Types float4, float8 (keys, filters, aggregates) Yes sift_col, sift_table SELECT f8, count(*) FROM t GROUP BY f8 A float key is one group per value as float8_eq has it: -0 and 0 together, every NaN together. fixture:t_float_keys,fixture:t_float8_group_values,fixture:t_float8_nan_inf agg SELECT f4, f8, count(*) FROM cv_types GROUP BY 1, 2 T03 Types numeric(p, s) with p <= 18 Yes sift_col SELECT n2, count(*), sum(n2) FROM t GROUP BY n2 Stored as integers scaled by 10^s: keys, predicates, sums, averages, min, max, distinct counts, expressions (+ - *, round, floor, casts to integer) are exact. numeric(p<=15) cast to float8 also feeds sum, avg and percentile_cont. fixture:n_numeric_key,fixture:n_numeric_pred,fixture:n_numeric_float,test:agg_numeric agg SELECT grp, n2, count(*), sum(n2), sum(n15::float8) FROM cv_types GROUP BY 1, 2 T04 Types numeric(p > 18), unconstrained numeric, negative scale, numeric(p, s) with s > 18 Yes sift_col SELECT grp, count(*) FROM t WHERE n38 > 100 GROUP BY grp Stored as lanes of 128 bits or as a canonical image. Every shape is PostgreSQL's own function on the value: conjuncts (also under OR, NOT, CASE, FILTER), keys, count, min, max, sum, avg, DISTINCT and expressions. NaN and the infinities compare as PostgreSQL orders them, 1.0 and 1.00 are one group that shows the first spelling seen (a hash aggregate's); serial for a key over such a column. fixture:w_num38_range,fixture:w_num38_nan,fixture:w_image_ties,fixture:w_image_order,fixture:w_neg_scale,fixture:w_key_n38,fixture:w_agg_numeric,fixture:w_tree_or,test:col_agg_wide,test:col_agg_wide_keys agg SELECT grp, count(*), sum(i4), sum(n38), max(nfree) FROM cv_types WHERE n38 > 100 AND nfree <> 3 GROUP BY grp 11:`numeric(p>18)`, `money`, `interval` T05 Types CREATE INDEX ... USING sift_col on a point, tsvector, json or xml column Error 0A000 at CREATE INDEX; leave the column out of the index sift_col CREATE INDEX ON t USING sift_col (point_col) The hint lists the types: smallint, integer, bigint, boolean, date, time, timetz, timestamp, timestamptz, interval, numeric, money, real, double precision, text, varchar, char(n), name, bytea, uuid, inet, cidr, macaddr, macaddr8, bit, varbit, enums, arrays, ranges and jsonb with paths. A type with no equality or no operator class (point, tsvector, xml, json) has no leaf form. test:col_index_wide error:0A000 -- pre: DROP TABLE IF EXISTS cv_pt;\n-- pre: CREATE TABLE cv_pt (id int, p point);\nCREATE INDEX cv_x ON cv_pt USING sift_col (id, p) T06 Types text, varchar (keys, filters, min/max, distinct; strings of any length) Yes sift_col, sift_table SELECT vc, count(*), min(tx) FROM t GROUP BY vc A dictionary per segment; strings over 255 bytes are stored inline per unit. Filters on the encoded form. fixture:t_text_long,fixture:t_text_long_filter,test:col_agg_text agg SELECT vc, count(*), min(tx), max(tx) FROM cv_types GROUP BY vc T07 Types char(n), name Yes a bpchar without a length is accepted but no predicate or group is pushed on it sift_col SELECT ch, count(*) FROM t GROUP BY ch test:col_index_chars agg SELECT ch, count(*) FROM cv_types GROUP BY ch T08 Types bytea Yes sift_col, sift_table SELECT ba, count(*) FROM t GROUP BY ba Stored as the image of the value; keys, count(DISTINCT), min, max and conjuncts. fixture:m_bytea,fixture:m_bytea_filter,fixture:m_bytea_minmax,test:col_agg_types32 agg SELECT ba, count(*), count(DISTINCT u) FROM cv_types GROUP BY ba 12:`time`, `timetz`, enums, `uuid`, `bytea`, `inet`, `macaddr`, `bit`, arrays, ranges T09 Types boolean Yes sift_col, sift_table SELECT b, count(*) FROM t GROUP BY b fixture:g_stats_numeric agg SELECT b, count(*) FROM cv_types GROUP BY b T10 Types date, timestamp, timestamptz (keys, filters, min/max, date_trunc of a date) Yes sift_col, sift_table SELECT d, count(*), min(ts), max(tstz) FROM t GROUP BY d date_trunc of a timestamp or timestamptz as a key: see the group keys. fixture:z_zone_ranges,fixture:month_series agg SELECT d, count(*), min(ts), max(tstz) FROM cv_types GROUP BY d T11 Types time Yes sift_col, sift_table SELECT tm, count(*) FROM t GROUP BY tm An int64 of microseconds; timetz is T33. fixture:m_time_key,fixture:m_time_filter,test:col_agg_types32 agg SELECT tm, count(*), max(tm) FROM cv_types GROUP BY tm 12:`time`, `timetz`, enums, `uuid`, `bytea`, `inet`, `macaddr`, `bit`, arrays, ranges T12 Types interval Yes sift_col, sift_table SELECT iv, count(*), max(iv), sum(iv) FROM t GROUP BY iv Equality and order are those of PostgreSQL (1 month = 30 days = 720 hours); keys, count, min, max, sum, avg, DISTINCT and conjuncts under OR or FILTER call its functions on the value, and a group shows the first spelling seen. fixture:w_interval_ties,fixture:w_interval_order,fixture:w_key_interval,fixture:w_agg_interval,fixture:g_ordered_set agg SELECT iv, count(*), max(iv), sum(iv) FROM cv_types WHERE iv >= interval '10 days' GROUP BY iv 11:`numeric(p>18)`, `money`, `interval` T13 Types uuid Yes sift_col, sift_table SELECT u, count(*) FROM t GROUP BY u The 16 bytes; keys, count(DISTINCT) and conjuncts (uuid has no min or max). fixture:m_uuid,fixture:m_uuid_filter,test:col_agg_types32 agg SELECT u, count(*) FROM cv_types GROUP BY u 12:`time`, `timetz`, enums, `uuid`, `bytea`, `inet`, `macaddr`, `bit`, arrays, ranges T14 Types money Yes sift_col, sift_table SELECT m, count(*), sum(m), max(m) FROM t GROUP BY m The integer of the smallest unit, so it does not depend on lc_monetary. Keys, min, max, sum (an overflow raises as the heap's) and conjuncts under OR or FILTER. fixture:w_money_cmp,fixture:w_money_in,fixture:w_key_money,fixture:w_agg_money agg SELECT m, count(*), sum(m), max(m) FROM cv_types WHERE m > '10.00' GROUP BY m 11:`numeric(p>18)`, `money`, `interval` T26 Types A boolean tree (OR, NOT, CASE, FILTER), a key, an aggregate or an expression of numeric(p > 18), unconstrained numeric, money or interval Yes sift_col SELECT m, count(*) FILTER (WHERE n38 > 5 OR iv > interval '1 day'), sum(n38) FROM t GROUP BY m Tested on PostgreSQL's own functions and compared with the normal plan by every fixture; a key or an aggregate over such a column runs serial, and the planner keeps the normal plan for a column with a distinct value per row. fixture:w_wide_group_numeric,fixture:w_wide_minmax,fixture:w_wide_sum,fixture:w_tree_filter,fixture:w_tree_case,fixture:w_tree_exact,test:col_agg_wide_keys agg SELECT m, count(*) FILTER (WHERE n38 > 5 OR iv > interval '1 day'), sum(n38) FROM cv_types GROUP BY m 11:`numeric(p>18)`, `money`, `interval` T15 Types enum Yes sift_col SELECT e, count(*) FROM t GROUP BY e The OID of the value is stored and the order is the catalog's when the query runs, so a value added with ALTER TYPE ... ADD VALUE BEFORE/AFTER after the index was built compares right. fixture:m_enum_key,fixture:m_enum_cmp,fixture:m_enum_in,test:col_agg_types32 agg SELECT e, count(*), max(e) FROM cv_types GROUP BY e 12:`time`, `timetz`, enums, `uuid`, `bytea`, `inet`, `macaddr`, `bit`, arrays, ranges T16 Types arrays (int[], text[], numeric[]), ranges and multiranges Yes sift_col, sift_table SELECT ia, count(*) FROM t GROUP BY ia Stored as the image of the value; equality and every operator are PostgreSQL's (@>, &&, <@ ...). jsonb arrays of objects are the other array form (next rows). fixture:m_array,fixture:m_array_ops,fixture:m_array_numeric,fixture:m_range,fixture:m_range_ops,test:col_agg_types32 agg SELECT ia, count(*) FROM cv_types GROUP BY ia 12:`time`, `timetz`, enums, `uuid`, `bytea`, `inet`, `macaddr`, `bit`, arrays, ranges T33 Types inet, cidr, macaddr, macaddr8, timetz, bit, varbit Yes sift_col SELECT ip, count(*) FROM t GROUP BY ip Stored as the image of the value; keys, count(DISTINCT), conjuncts (= <> < > <<= >>= &&), min and max (inet, cidr and timetz have them) and expressions (host, family, masklen) are PostgreSQL's own functions on the value, so '10.0.0.1/24' and '10.0.0.1/32', or '12:00+01' and '11:00+00', are told apart as PostgreSQL does. bit and varbit have no hash operator class (PostgreSQL groups them by sorting): a GROUP BY or DISTINCT on a column of one groups by the image of the value (equal values have equal images), an expression of one keeps the normal plan. A cidr key that the same query also reads as inet keeps it too. fixture:n_net_inet_key,fixture:n_net_inet_cmp,fixture:n_net_inet_minmax,fixture:n_net_cidr_key,fixture:n_net_mac,fixture:n_net_timetz,fixture:n_net_bit,fixture:n_net_bit_key,fixture:n_net_bit_distinct,fixture:n_net_bit_distinct_both,test:col_agg_types33 agg SELECT ip, count(*), max(ip), count(DISTINCT tz) FROM cv_net GROUP BY ip 12:`time`, `timetz`, enums, `uuid`, `bytea`, `inet`, `macaddr`, `bit`, arrays, ranges T17 Types jsonb members declared in the paths option, typed (->>, ->, #>> and casts) Yes sift_col, sift_table SELECT attributes->>'channel', count(*) FROM orders GROUP BY 1 Declare each path with a type: WITH (paths = 'attributes->channel text, metrics.* float8, ...'). A member of another type than declared is stored as an overflow: a recordset column reads it again from the heap (T37), any other read keeps the normal plan. fixture:f_scalar_paths,fixture:metric_key agg SELECT attributes->>'channel', count(*) FROM orders GROUP BY 1 10:`jsonb` paths: `col->>'k'`, `jsonb_to_recordset(col)` T18 Types jsonb member that is not declared in the paths option No the index does not hold it sift_col SELECT attributes->>'device', count(*) FROM orders GROUP BY 1 Add it to paths and REINDEX. script:coverage plain SELECT attributes->>'device', count(*) FROM orders GROUP BY 1 T19 Types jsonb array of objects through jsonb_to_recordset, jsonb_array_elements (one level) Yes sift_col, sift_table SELECT i.category, count(*), sum(i.qty) FROM orders o, jsonb_to_recordset(o.items) i(category text, qty int) GROUP BY 1 One lateral unnest per query; elements of one row keep the same-element semantics in filters. fixture:item_category_counts,fixture:exists_same_element agg SELECT i.category, count(*), sum(i.qty) FROM orders o CROSS JOIN LATERAL jsonb_to_recordset(o.items) AS i(category text, qty int) GROUP BY 1 10:`jsonb` paths: `col->>'k'`, `jsonb_to_recordset(col)` T20 Types jsonb object with dynamic keys (jsonb_each, jsonb_each_text, metrics ->> 'key') Yes sift_col, sift_table SELECT kv.key, avg((kv.value #>> '{}')::float8) FROM orders o, jsonb_each(o.metrics) kv GROUP BY 1 Declared as key.* type. fixture:metric_key,fixture:metrics_summary agg SELECT kv.key, count(*) FROM orders o CROSS JOIN LATERAL jsonb_each(o.metrics) kv GROUP BY 1 10:`jsonb` paths: `col->>'k'`, `jsonb_to_recordset(col)` T21 Types jsonb arrays nested deeper than one level, or two arrays unnested together Error 0A000 at CREATE INDEX; keep one level of arrays sift_col, sift_table CREATE INDEX ... WITH (paths = 'j.a[]{b[]{c int4}}') A query with two unnests, or an element compared with another array, keeps the normal plan (see the join rows). fixture:t_exists_two_arrays error:0A000 CREATE INDEX cv_x ON cv_types USING sift_col (j) WITH (paths = 'j.a[]{b[]{c int4}}') T22 Types jsonb column with no paths Yes the paths are read from a sample of the documents at CREATE INDEX (sift.suggest_paths); a table where no key qualifies is refused (22023) sift_col CREATE INDEX ON t USING sift_col (j) test:col_paths_auto ok CREATE INDEX cv_x ON cv_types USING sift_col (j) T23 Types numeric NaN in an indexed column Yes sift_col, sift_table SELECT g, sum(n) FROM tbl GROUP BY g The index stores NaN as NULL with an overflow flag and the scan gives the row the value PostgreSQL orders it by (NaN equals NaN and is greater than every number): min and max, sums and averages, counts, DISTINCT, keys, ORDER BY, comparisons and expressions return the rows of the normal plan. A sift_table column group gives the same rows (`tbl_topn`: `num_desc`, `num_nan`). test:agg_numeric_nan,test:tbl_topn agg -- guc: seq_page_cost = 10000\n-- guc: random_page_cost = 10000\nSELECT g, sum(n) FROM cv_nan GROUP BY g T24 Types sift_table: scalar columns (integers, floats, text, varchar, date, timestamps, boolean, numeric(p <= 18), char(n), enum) Yes sift_table SELECT grp, e, count(*), sum(n2) FROM t GROUP BY grp, e The engine keeps every type exactly; the aggregate scan reads these. test:tbl_agg agg SELECT grp, ch, e, count(*), sum(i4), sum(n2), min(d), max(tx) FROM cv_tt GROUP BY grp, ch, e T25 Types sift_table: a key, aggregate, conjunct or expression of numeric(p > 18) and unconstrained numeric, uuid, interval, bytea, money, time, inet, arrays Yes sift_table SELECT u, count(*) FROM t GROUP BY u Read through a dictionary of the values, as on a sift_col index (T08, T11, T13, T16). A numeric key shows the spelling of the first row of the group (1.0 or 1.00), as PostgreSQL does. test:tbl_agg_images,test:tbl_agg agg SELECT grp, count(DISTINCT u), min(iv), sum(m), max(tm), count(DISTINCT ba) FROM cv_tt GROUP BY grp T36 Types sift_table: a WHERE conjunct, OR or FILTER on numeric(p > 18), unconstrained numeric, money or interval Yes sift_table SELECT grp, count(*) FROM t WHERE m > '10.00' GROUP BY grp The comparison is the type's own, as on a sift_col index (T04, T12, T14); a column added by ALTER TABLE is tested by its default. test:tbl_agg_wide,test:tbl_agg_images,fixture:w_money_cmp,fixture:w_interval_order agg SELECT grp, count(*), sum(i4) FROM cv_tt WHERE n38 > 100 AND m > '10.00' AND iv >= interval '10 days' GROUP BY grp T34 Types sift_table: jsonb paths declared with sift.table_set_paths (->>, ->, casts, jsonb_to_recordset and jsonb_array_elements of an array of objects) Yes sift_table SELECT sift.table_set_paths('t', 'doc', 'doc->ch text, doc->w int4, doc->items[]{k int4, p float8}') The paths grammar of a sift_col index. Promotion, merge and VACUUM FULL write a leaf per path next to the exact bytes, the write buffer is shredded as it is read. The plan is the normal one until every segment has the leaves (VACUUM FULL after declaring). A row that holds another JSON type under a declared path keeps the normal plan for the queries that read that path (the same rule as the overflow of a sift_col index). No format change; pg_dump carries the declaration. test:tbl_paths agg SELECT doc->>'a', count(*), sum((doc->>'m')::float8) FROM cv_tc GROUP BY 1 T37 Types jsonb documents that hold what a typed leaf cannot store (a numeric string, an array or scalar root, a number in a text path), read through jsonb_to_recordset Yes sift_col index; the recordset columns are bool, int2, int4, int8, float4, float8 or text sift_col SELECT o.k, count(*), sum(o.v) FROM docs d CROSS JOIN LATERAL jsonb_to_recordset(d.doc -> 'l') AS o(k text, v float8) GROUP BY o.k The flagged rows are read again from the heap by the scan and read as jsonb_to_recordset reads them (the text of a scalar through the input function of the column, a key of an array is NULL), so the answer is the normal plan's, errors included: the error is raised only for a row that reaches the function. `EXPLAIN ANALYZE` shows `Rows Read Again From The Heap`. A string `NaN` or `Infinity` converts; the string `-0` still raises the scan's `0A000` error. A `jsonb_each` member read through `CAST(value AS float8)` on a `col.* float8` path is read again the same way and raises the cast's own error (`cannot cast jsonb string to type double precision`) for the rows that reach it. A flagged leaf read in another way (a key of an object leaf, jsonpath, `@>`, `jsonb_array_length`), a member of one key and a sift_table keep the normal plan, so the results are PostgreSQL's: under a cast a numeric string converts and any other value raises its own error, never a NULL. No format change. test:col_dirty_recheck,test:col_dirty_recheck_parallel,test:agg_member_cast_dirty,test:agg_vleaf_overflow,fixture:rc_recordset,fixture:rc_recordset_filtered,fixture:rc_recordset_error,fixture:vd_recordset_values,fixture:vd_member_text_cast_values,fixture:vd_scalar_cast_values,fixture:vd_array_length_values agg SELECT o.k, count(*), sum(o.v) FROM t_rc r CROSS JOIN LATERAL jsonb_to_recordset(r.doc -> 'l') AS o(k text, v float8, n int4, ok bool) WHERE r.grp <> 6 GROUP BY o.k T35 Types sift_table: a jsonb path declared after the data was written, or a table rewritten by ALTER TABLE ... TYPE No the older segments have no leaf of the path until VACUUM FULL rewrites them sift_table SELECT doc->>'a', count(*) FROM cv_tq GROUP BY 1 Run VACUUM FULL after sift.table_set_paths. Nothing is wrong meanwhile: the normal plan gives the same result. test:tbl_paths plain SELECT doc->>'a', count(*) FROM cv_tq GROUP BY 1 JT09 Types jsonb_typeof(path) of an astext path Yes sift_col SELECT jsonb_typeof(doc->'a') AS ty, count(*) FROM cv_typeof GROUP BY 1 Every form above works on an astext leaf: the chunk keeps the kind (string, number, boolean) of each value of the text, a JSON null apart from a missing key. A string that looks like a number is a string. fixture:k_typeof_astext,fixture:k_typeof_astext_kinds,fixture:k_typeof_astext_complement,test:agg_jsonb_typeof agg SELECT jsonb_typeof(doc->'a') AS ty, count(*) FROM cv_typeof GROUP BY 1 JT05 Types jsonb_typeof(path) as a group key Yes sift_col SELECT jsonb_typeof(doc->'m') AS ty, count(*) FROM cv_typeof GROUP BY 1 The five types, 'null' for a JSON null and a NULL group for the paths with no value. The path is a typed leaf of the index. fixture:k_typeof_group_key,test:agg_jsonb_typeof agg SELECT jsonb_typeof(doc->'m') AS ty, count(*) FROM cv_typeof GROUP BY 1 JT06 Types jsonb_typeof(path) in a branch of OR, NOT Yes sift_col SELECT count(*) FROM cv_typeof WHERE jsonb_typeof(doc->'m') = 'null' OR grp = 2 Beside predicates of other columns and other paths, with PostgreSQL's NULL semantics (a missing key is not <>). fixture:k_typeof_or,test:agg_jsonb_typeof agg SELECT count(*) FROM cv_typeof WHERE jsonb_typeof(doc->'m') = 'null' OR grp = 2 JT07 Types jsonb_typeof(path) in an aggregate FILTER or a CASE Yes sift_col SELECT count(*) FILTER (WHERE jsonb_typeof(doc->'m') = 'number') AS nums, count(*) FILTER (WHERE jsonb_typeof(doc->'m') IS NULL) AS missing FROM cv_typeof Also as a CASE argument or key and under a function of the type. Reading the values of the same path in that query keeps the normal plan (JT08). fixture:k_typeof_filter,fixture:k_typeof_case,test:agg_jsonb_typeof agg SELECT count(*) FILTER (WHERE jsonb_typeof(doc->'m') = 'number') AS nums, count(*) FILTER (WHERE jsonb_typeof(doc->'m') IS NULL) AS missing FROM cv_typeof JT08 Types jsonb_typeof(path) in a FILTER or CASE guarding a read of the values of the same path No the leaf cannot hold the strings among its numbers, which PostgreSQL's guard skips; use the conjunct form (JT01) sift_col SELECT sum((doc->>'m')::float8) FILTER (WHERE jsonb_typeof(doc->'m') = 'number') FROM cv_typeof The normal plan, the same result. fixture:k_typeof_not_pushed,test:agg_jsonb_typeof plain SELECT sum((doc->>'m')::float8) FILTER (WHERE jsonb_typeof(doc->'m') = 'number') FROM cv_typeof JT01 Types jsonb_typeof(path) = 'type' and IN a list of types, on a typed scalar path Yes sift_col SELECT count(*), sum((doc->>'m')::float8) FROM cv_typeof WHERE jsonb_typeof(doc->'m') = 'number' The path is a typed leaf of the index (not a key of a map or list). The rows of another type are dropped without reading their value, so the strings among the numbers do not raise; a kept row whose value the leaf cannot hold raises only where the query reads the leaf. fixture:k_typeof_number,fixture:k_typeof_string,fixture:k_typeof_boolean,test:agg_jsonb_typeof agg SELECT count(*), sum((doc->>'m')::float8) FROM cv_typeof WHERE jsonb_typeof(doc->'m') = 'number' JT02 Types jsonb_typeof(path) <> 'type', NOT IN a list of types, NOT of an equality Yes sift_col SELECT count(*) FROM cv_typeof WHERE jsonb_typeof(doc->'m') <> 'number' The other types, but not a path with no value: PostgreSQL gives NULL for a missing key, which no comparison keeps. Counting the rows by their types reads no value, so the strings of a number leaf do not raise. fixture:k_typeof_complement,fixture:k_typeof_not_in,test:agg_jsonb_typeof agg SELECT count(*) FROM cv_typeof WHERE jsonb_typeof(doc->'m') <> 'number' JT03 Types jsonb_typeof(path) = 'null' (a JSON null) Yes sift_col SELECT count(*) FROM cv_typeof WHERE jsonb_typeof(doc->'m') = 'null' A JSON null is not a missing key: the chunk keeps which of the two a NULL slot is (column format minor 4). A segment of minor 3 cannot say and raises 0A000 with a REINDEX hint. fixture:k_typeof_json_null,test:agg_jsonb_typeof,test:col_tags agg SELECT count(*) FROM cv_typeof WHERE jsonb_typeof(doc->'m') = 'null' JT04 Types jsonb_typeof(path) IS NULL and IS NOT NULL Yes sift_col SELECT count(*) FROM cv_typeof WHERE jsonb_typeof(doc->'m') IS NULL IS NULL is a path with no value (a missing key, a NULL document), not a JSON null. fixture:k_typeof_is_null,fixture:k_typeof_is_not_null,test:agg_jsonb_typeof agg SELECT count(*) FROM cv_typeof WHERE jsonb_typeof(doc->'m') IS NULL JT10 Types jsonb_typeof(path) on a sift_table with declared paths Yes sift_table SELECT count(*) FROM cv_tp WHERE jsonb_typeof(doc->'m') = 'number' Declare the path with sift.table_set_paths (T34). The conjunct forms of JT01 to JT04 and the key of JT05 run as on a sift_col index, also over the rows that hold another type under the path. test:tbl_paths agg SELECT count(*) FROM cv_tp WHERE jsonb_typeof(doc->'m') = 'number' T31 Types The value of jsonb_each cast to a number or boolean (kv.value::float8), not through its text Yes sift_col, sift_table SELECT kv.key, avg(kv.value::float8) FROM orders o, jsonb_each(o.metrics) kv GROUP BY 1 As (kv.value #>> '{}')::float8: the declared type of the wildcard path. fixture:j_jsonb_value_cast agg ~ metrics\.\*\.value @j_jsonb_value_cast T32 Types The subscript of a jsonb column with a text key (x['k'], chained) as x -> 'k' Yes sift_col, sift_table SELECT attributes['channel'] #>> '{}', avg((metrics['m_total'])::float8) FROM orders GROUP BY 1 In casts, as #>> (its text) and as a key or a filter. An integer subscript, and a comparison of the jsonb value itself (x['k'] = '"a"'), are not read from the index. fixture:j_jsonb_subscript agg ~ avg\(metrics @j_jsonb_subscript T38 Types The integer cast of a jsonb member held as float8 ((x -> 'k')::int, ::bigint), in a filter, a group key or an aggregate Yes before PostgreSQL 18 a leaf that may hold a JSON null keeps the normal plan (the cast of a JSON null raises there) sift_col SELECT (metrics -> 'm_total')::int, count(*) FROM orders WHERE (metrics -> 'quality')::int >= 4 GROUP BY 1 The cast is PostgreSQL's own over the jsonb the stored float stands for: it rounds as numeric does, a missing key and a JSON null are NULL, and an out of range member raises `integer out of range` for the rows that reach it. A member that is no number keeps the normal plan. fixture:j_jsonb_int_cast,test:agg_jsonb_int_cast agg ~ to_jsonb\(float8 @j_jsonb_int_cast T39 Types A typed scalar path of a jsonb column (doc.v float8, int4, int8, bool) that holds what the leaf cannot store (a numeric string, a boolean, a container), cast as (doc ->> 'v')::T or (doc -> 'v')::float8 in an aggregate (also its FILTER), a group key or a WHERE condition, or the member of an element of jsonb_array_elements (p ->> 'score')::float8 Partly sift_col index; the condition is a comparison with a constant, an IN list of integers or a NULL test of the cast, and PostgreSQL tests every other condition of the WHERE before it (cheaper, or as cheap and written first) sift_col SELECT grp, count(*) FROM docs WHERE (doc ->> 'k')::float8 > 3 AND grp <> 2 GROUP BY grp The flagged rows are read again from the heap by the scan and answer as the normal plan does, errors included: a numeric string is converted, a value SQL cannot convert raises the error of the cast for a row the other conditions keep, and never for one they drop. `dirty_order` names the queries left to the normal plan because PostgreSQL may test the cast before a condition that would drop the row (a costlier condition, one written before it with the same cost, one that may raise too, two casts of paths that hold bad values, an OR); `dirty_units` the other shapes of a condition or FILTER (an IN list of floats, an expression of the cast, a bare boolean, an AND in the FILTER), a cast under CASE, a FILTER that may raise, a condition or a FILTER on the elements of a list, and any other read of the path (a join, a sub-select). The cast in the argument of an aggregate with a FILTER raises only for the rows the FILTER keeps. Past about 14% of the rows read again the normal plan is cheaper and the planner keeps it: a row read again costs 1.6 to 2.9 microseconds, so the scan is never within 1.3x of the scan of clean data. test:agg_scalar_dirty,test:agg_scalar_dirty_where,test:agg_scalar_dirty_filter,test:agg_scalar_dirty_elems,test:agg_scalar_dirty_share,fixture:vd_scalar_where_values,fixture:vd_scalar_where_dropped,fixture:vd_scalar_where_error_text,fixture:vd_scalar_where_jsonb_cast_values,fixture:vd_scalar_filter_values,fixture:vd_scalar_elems_values,fixture:vd_scalar_elems_dropped,fixture:vd_scalar_elems_error_text,fixture:vd_scalar_filter_dropped,fixture:vd_scalar_filter_error_text,fixture:vd_scalar_cast_values agg SELECT grp, count(*) FROM t_vd WHERE (doc ->> 'k')::float8 > 1 AND grp IN (0, 1, 5, 7, 8) GROUP BY grp T40 Types cardinality(x), array_length(x, 1) and x IS [NOT] NULL of an array column, as a filter, in FILTER, CASE, a group key or an aggregate Yes sift_col index built with array_lengths on (the default of CREATE INDEX and REINDEX since 3.6); an index built before it, or with array_lengths = off, reads the images sift_col SELECT count(*) FROM t WHERE ia IS NOT NULL AND cardinality(ia) > 0 Read from two integer columns of the array, without decoding it: the presence and length tests never touch the images. array_length(x, 2), @>, && and unnest still read the arrays (T16). test:col_array_lengths,test:col_agg_types32 agg SELECT count(*) FROM cv_types WHERE ia IS NOT NULL AND cardinality(ia) > 0 K01 Group keys GROUP BY a column (dictionary, integer, date, boolean, float, numeric(p<=18)) Yes sift_col, sift_table SELECT status, count(*) FROM orders GROUP BY status Up to 8 group columns; 7 with grouping sets. fixture:f_group_source agg SELECT tenant_id, source, status, lang, country_code, count(*) FROM orders GROUP BY 1, 2, 3, 4, 5 05:`GROUP BY x`, `GROUP BY expr`, `ROLLUP`, `CUBE` K02 Group keys GROUP BY an expression of one column: lower(), CASE, date_trunc of a date, x % 7, extract() Yes sift_col, sift_table SELECT date_trunc('month', placed_at::timestamp)::date, count(*) FROM orders GROUP BY 1 Evaluated by PostgreSQL once per distinct value of the column. fixture:f_group_exprs agg @f_group_exprs 05:`GROUP BY x`, `GROUP BY expr`, `ROLLUP`, `CUBE` K03 Group keys GROUP BY an expression of two or three columns Yes sift_col, sift_table SELECT CASE WHEN lang = 'en' THEN source ELSE status END, count(*) FROM orders GROUP BY 1 Evaluated once per distinct tuple of the values, cached across units. fixture:f_multi_leaf_keys agg @f_multi_leaf_keys 05:`GROUP BY x`, `GROUP BY expr`, `ROLLUP`, `CUBE` K04 Group keys GROUP BY date_trunc, extract of a timestamp or timestamptz, or CASE over numeric Yes sift_col, sift_table SELECT date_trunc('month', ts), count(*) FROM t GROUP BY 1 Integer, float, date, timestamp and numeric expressions with no kernel are evaluated by PostgreSQL per distinct tuple of values. fixture:e_timestamp_expr,fixture:n_numeric_pg_keys agg SELECT date_trunc('month', ts), extract(hour FROM ts)::int, count(*) FROM cv_types GROUP BY 1, 2 expr-timestamp 05:`GROUP BY x`, `GROUP BY expr`, `ROLLUP`, `CUBE` K05 Group keys More than 8 group columns (tested with 24) Yes the grouping and arm masks have a bit per key (SIFT_AGG_MAX_KEYS, 32), the set number included; a query with more keys keeps the normal plan sift_col, sift_table GROUP BY a, b, c, d, e, f, g, h, i fixture:k_keys_9,test:agg_group_key_width agg @k_keys_9 K06 Group keys One group per row (high cardinality) Yes sift_col, sift_table SELECT id, count(*) FROM orders GROUP BY id Direct array up to 2^20 groups, hash table beyond. fixture:per_order_sets_hash agg SELECT id, count(*), sum(total) FROM orders GROUP BY id 05:`GROUP BY x`, `GROUP BY expr`, `ROLLUP`, `CUBE` K07 Group keys float4 / float8 keys Yes sift_col, sift_table SELECT f8, count(*) FROM t GROUP BY f8 -0 and 0 are one group, every NaN is one group. fixture:t_float_keys,fixture:t_float_keys_items agg @t_float_keys 05:`GROUP BY x`, `GROUP BY expr`, `ROLLUP`, `CUBE` K08 Group keys GROUPING SETS, ROLLUP, CUBE, GROUPING() Yes sift_col, sift_table SELECT tenant_id, status, count(*) FROM orders GROUP BY CUBE (tenant_id, status) One scan for all sets. fixture:t_sets_cube_having,fixture:t_sets_empty,fixture:t_sets_duplicate agg SELECT tenant_id, status, count(*) FROM orders GROUP BY CUBE (tenant_id, status) 05:`GROUP BY x`, `GROUP BY expr`, `ROLLUP`, `CUBE` K09 Group keys ROLLUP or CUBE of more than 7 columns Yes the set number is a key of the 32 the masks hold sift_col, sift_table GROUP BY ROLLUP (a, b, c, d, e, f, g, h) fixture:k_rollup_8 agg @k_rollup_8 K10 Group keys Key that is a volatile expression No PostgreSQL evaluates it per row, the scan once per value sift_col, sift_table GROUP BY random() > 0.5 fixture:t_key_expr_volatile plain @t_key_expr_volatile 13:`RIGHT JOIN`, `SELECT DISTINCT`, volatile functions K11 Group keys Key over a column that is not in the index No the scan reads only indexed columns sift_col SELECT left(memo, 3), count(*) FROM orders GROUP BY 1 script:coverage plain SELECT left(memo, 3), count(*) FROM orders GROUP BY 1 K12 Group keys SELECT DISTINCT of columns or expressions the index holds Yes sift_col SELECT DISTINCT status FROM orders A group-by with no aggregate: the distinct stage of the planner takes the scan; ORDER BY and LIMIT stay above it. Not for a SELECT DISTINCT over a grouped or windowed query. fixture:k_select_distinct_one,fixture:k_select_distinct_multi,test:agg_distinct agg SELECT DISTINCT status FROM orders 13:`RIGHT JOIN`, `SELECT DISTINCT`, volatile functions K13 Group keys Key or filter expression over an integer column with a wide range of values (x % 7 of an int8) Yes sift_col, sift_table SELECT i8 % 7, count(*) FROM t GROUP BY 1 Evaluated by PostgreSQL per distinct value that occurs, not per value of the range. fixture:e_int8_expr agg SELECT i8 % 7, count(*) FROM cv_types WHERE i8 % 11 <> 3 GROUP BY 1 expr-int 05:`GROUP BY x`, `GROUP BY expr`, `ROLLUP`, `CUBE` K14 Group keys DISTINCT ON (columns) Yes sift_col, sift_table SELECT DISTINCT ON (tenant_id) tenant_id, id, placed_at FROM orders ORDER BY tenant_id, placed_at DESC, id The first row of each group by the ORDER BY is kept while the rows pass, with no sort; the select list is columns and expressions of them. At most 4 sort columns after the ON columns, and the best rows fit the memory of a hash aggregate. Rows that tie in the ORDER BY keep the first one the scan meets (SQL leaves it unspecified). fixture:do_latest,fixture:do_earliest,fixture:do_mixed_keys,fixture:do_two_keys,fixture:do_join,fixture:do_t_misc,test:agg_distinct_on,fixture:do_ties agg @do_latest 13:`RIGHT JOIN`, `SELECT DISTINCT`, volatile functions K15 Group keys Constant or stable expression that reads no column in the GROUP BY list (GROUP BY 1, 2 over a literal and a key; alone) Yes sift_col, sift_table SELECT 'cur'::text AS label, country_code, count(*) FROM orders GROUP BY 1, 2 A constant next to another key adds no group: it is dropped from the keys and output as it is. Alone, or as a stable expression (current_setting(...), now()), it is one group, evaluated once; a scan with no rows gives no group, as in the normal plan. fixture:k_group_const,fixture:k_const_stable,fixture:k_const_literal,fixture:k_const_empty,fixture:k_const_with_key,fixture:k_const_join agg @k_group_const K16 Group keys sift_table: the expression of a stored generated column (GENERATED ALWAYS AS (date_trunc('month', ts)::date) STORED) as a key or in a filter Yes sift_table SELECT date_trunc('month', ts)::date, sum(v) FROM t GROUP BY 1 The column is read, with its pre-aggregates, where the query writes the expression; a sift_col index does the same for an index expression. The generation expression must be the query's (same functions and casts). test:tbl_generated agg ~ Group Keys: mon SELECT date_trunc('month', ts)::date, sum(v) FROM cv_gt GROUP BY 1 K17 Group keys DISTINCT ON (columns) whose values cannot be kept No a volatile or array value, a sort or value of a joined table, or the groups would not fit the hash memory sift_col, sift_table SELECT DISTINCT ON (tenant_id) tenant_id, ARRAY[id] FROM orders ORDER BY tenant_id, placed_at DESC The normal plan stays; select the columns, not an array of them. fixture:do_plain_cap,fixture:do_plain_array,fixture:do_plain_volatile plain @do_plain_array 13:`RIGHT JOIN`, `SELECT DISTINCT`, volatile functions K18 Group keys GROUPING SETS, ROLLUP, CUBE over keys that are expressions of a sub-select or a CTE (the sub-select is pulled up and PostgreSQL hands each key to the grouping step as a placeholder) Yes sift_col, sift_table SELECT s, count(*) FROM (SELECT upper(status) AS s FROM orders) p GROUP BY ROLLUP (s) GROUPING(), the NULL rows and the order of the sets are PostgreSQL's. test/sql/agg_eligibility_survey.sql lists the other ordinary shapes the scan answers and the ones it leaves to the normal plan, with the reason. test:agg_grouping_set_keys,test:agg_eligibility_survey agg SELECT s, count(*) FROM (SELECT upper(status) AS s FROM orders) p GROUP BY ROLLUP (s) 05:`GROUP BY x`, `GROUP BY expr`, `ROLLUP`, `CUBE` P01 Predicates Equality, range, BETWEEN, <> on integers, dates, text, numeric(p<=18) Yes sift_col, sift_table SELECT count(*) FROM orders WHERE tenant_id BETWEEN 5 AND 50 AND placed_at >= DATE '2024-01-01' Range tests on the encoded values; zone maps skip whole units on a clustered column. fixture:q_in_between,fixture:z_zone_ranges agg SELECT count(*) FROM orders WHERE tenant_id BETWEEN 5 AND 50 AND placed_at >= DATE '2024-01-01' AND status <> 'paid' 04:`WHERE x = 1`, `IN (...)`, `LIKE 'a%'`, `~ 're'`, `IS NULL` P02 Predicates IN and = ANY lists (any length; a numeric list of more than 64 constants is a hash set of the scaled values) Yes sift_col, sift_table SELECT count(*) FROM orders WHERE tenant_id IN (1, 2, 3) A parameter array: see the next rows. fixture:q_in_between agg SELECT count(*) FROM orders WHERE tenant_id IN (1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50) AND source = ANY(ARRAY['web','ios']) 04:`WHERE x = 1`, `IN (...)`, `LIKE 'a%'`, `~ 're'`, `IS NULL` P03 Predicates numeric compared with float8, numeric division in a filter Yes sift_col, sift_table SELECT count(*) FROM t WHERE n2 / (i2 + 1) > 20 Evaluated by PostgreSQL per distinct tuple of values. fixture:n_numeric_pg_filters agg SELECT count(*) FROM cv_types WHERE n2 / (i2 + 1) > 20 AND n2 > 10.5::float8 expr-numeric 04:`WHERE x = 1`, `IN (...)`, `LIKE 'a%'`, `~ 're'`, `IS NULL` P04 Predicates LIKE, ILIKE, prefix, regex (~), any boolean function of one text or integer column Yes sift_col SELECT count(*) FROM orders WHERE source LIKE 'market%' A lookup table over the distinct values of the column, evaluated by PostgreSQL. PostgreSQL estimates 1% of the rows for a filter on an expression (lower(col) IN (...)), so the normal plan can look cheaper than the scan: CREATE STATISTICS s (mcv) ON lower(col) FROM t; ANALYZE t gives the planner the real share. fixture:q_like_or_not agg SELECT count(*) FROM orders WHERE source LIKE 'market%' OR source ~ '^w' OR source ILIKE '%PLACE%' 04:`WHERE x = 1`, `IN (...)`, `LIKE 'a%'`, `~ 're'`, `IS NULL` P14 Predicates Text equality, <>, IN, NOT IN (NULL members too), LIKE 'prefix%' and C-collation ranges on a text or varchar column Yes sift_col, sift_table SELECT lang, count(*) FROM orders WHERE status = 'shipped' AND source IN ('web', 'pos') AND lang NOT IN ('en', 'xx') GROUP BY lang Constants, parameters and lists are compared on the stored strings in a deterministic collation; any other expression of the column is a lookup table over its distinct values (P04). fixture:q_in_between,fixture:t_text_inline_in agg SELECT lang, count(*), sum(tenant_id) FROM orders WHERE status = 'shipped' AND country_code <> 'US' AND source IN ('web', 'pos', 'nowhere') AND lang NOT IN ('en', 'xx') AND source LIKE 'w%' AND status COLLATE "C" >= 'p' GROUP BY lang 04:`WHERE x = 1`, `IN (...)`, `LIKE 'a%'`, `~ 're'`, `IS NULL` P15 Predicates The same text tests on strings over 255 bytes: =, IN, NOT IN, LIKE 'prefix%', C-collation ranges, count(DISTINCT s) Yes sift_col, sift_table SELECT grp, count(*), count(DISTINCT t) FROM docs WHERE t COLLATE "C" >= repeat('b', 300) GROUP BY grp Long strings are stored inline per unit and compared bytewise; the scan keeps the normal plan where the heap is faster, so the check raises the page costs. fixture:t_text_inline_range,fixture:t_text_inline_like,fixture:t_text_inline_in,fixture:t_text_long agg SELECT grp, count(*) AS n, count(DISTINCT t) AS d, min(id) AS lo FROM t_text WHERE t COLLATE "C" >= repeat('b', 300) AND t COLLATE "C" < repeat('c', 280) AND t LIKE repeat('b', 260) || '%' AND t NOT IN (repeat('a', 250) || '0', 'z') GROUP BY grp 04:`WHERE x = 1`, `IN (...)`, `LIKE 'a%'`, `~ 're'`, `IS NULL` P05 Predicates IS [NOT] NULL, AND, OR, NOT Yes sift_col, sift_table SELECT count(*) FROM orders WHERE status IS NULL OR NOT (tenant_id < 10 AND lang = 'en') fixture:q_null_tests agg SELECT count(*) FROM orders WHERE status IS NULL OR NOT (tenant_id < 10 AND (status = 'paid' OR lang = 'en')) 04:`WHERE x = 1`, `IN (...)`, `LIKE 'a%'`, `~ 're'`, `IS NULL` P06 Predicates Expressions of leaves: x + y < 20, total * 2 > 300 Yes sift_col, sift_table SELECT count(*) FROM orders WHERE tenant_id + id % 5 < 20 + - * of numeric and float leaves have kernels; the rest is evaluated by PostgreSQL per distinct tuple. fixture:n_numeric_expr_pred,fixture:f_multi_leaf_filter agg SELECT count(*) FROM orders WHERE tenant_id + id % 5 < 20 AND total * 2 > 300 04:`WHERE x = 1`, `IN (...)`, `LIKE 'a%'`, `~ 're'`, `IS NULL` P07 Predicates Parameters ($1) and stable functions (now(), current_date, a scalar sub-select) Yes sift_col, sift_table SELECT status, count(*) FROM orders WHERE placed_at > current_date - 3000 GROUP BY 1 Custom and generic plans. A qual with no column is a gate evaluated when the scan starts. fixture:f_gate_stable,fixture:p_generic_plan agg -- prepare: (int, text)\n-- args: 50, 'paid'\n-- guc: plan_cache_mode = force_generic_plan\nSELECT status, count(*) FROM orders WHERE tenant_id < $1 AND status <> $2 GROUP BY 1 04:`WHERE x = 1`, `IN (...)`, `LIKE 'a%'`, `~ 're'`, `IS NULL` P08 Predicates Volatile function in the WHERE (random(), clock_timestamp()) No evaluated per row by PostgreSQL, not a function of the data sift_col, sift_table WHERE tenant_id < random() * 100 fixture:t_volatile_qual plain @t_volatile_qual 13:`RIGHT JOIN`, `SELECT DISTINCT`, volatile functions P09 Predicates Predicate on a column that is not in the index No the scan reads only indexed columns sift_col SELECT count(*) FROM orders WHERE length(memo) > 10 script:coverage plain SELECT count(*) FROM orders WHERE length(memo) > 10 P10 Predicates Range predicates on ctid: = < <= > >= (either side) and BETWEEN against a constant TID Yes sift_col, sift_table SELECT grp, count(*) FROM t WHERE ctid >= '(0,1)' AND ctid < '(1000000,1)' GROUP BY grp The conjuncts of the WHERE that bound ctid make one range of TIDs: the scan reads the superblocks the range touches and tests the TIDs of the rows of the first and last block. A heap row is at the TID of the root of its HOT chain in the index and its ctid is the visible version's, so the rows of those two blocks are looked up in the heap (a sift_table row has the TID the engine gave it). Only a conjunct of the top level of the WHERE is a range, and a constant or a plan-time parameter: see the next row. A partitioned table has TIDs per partition and a top-N scan reads the rows another way: the normal plan stays. fixture:tr_in_block,fixture:tr_across_blocks,fixture:tr_across_sb,fixture:tr_many_sb,fixture:tr_between,fixture:tr_eq,fixture:tr_empty,fixture:tr_beyond_end,fixture:tr_offset_edges,fixture:tr_offset_zero,fixture:tr_with_filters,fixture:tr_join,fixture:tr_prepared_custom,fixture:tr_elements,test:agg_tid_range agg -- guc: enable_tidscan = off\nSELECT grp, count(*) FROM cv_types WHERE ctid >= '(0,1)' AND ctid < '(1000000,1)' GROUP BY grp P21 Predicates ctid <>, ctid in an expression, an OR, a list or a column; a parameter of a generic plan; ctid of a joined table No not a range of TIDs that the scan can read the superblocks of sift_col, sift_table SELECT grp, count(*) FROM t WHERE ctid <> '(0,1)' GROUP BY grp The conjunct is evaluated by PostgreSQL on the rows of the normal plan. A tid_range scan is the normal plan for a narrow range. fixture:tr_trap_ne,fixture:tr_trap_expr,fixture:tr_trap_or,fixture:tr_trap_in,fixture:tr_trap_group,fixture:tr_trap_key,fixture:tr_trap_ctid_cmp,fixture:tr_trap_filter,fixture:tr_trap_generic,fixture:tr_trap_joined,test:agg_tid_range plain SELECT grp, count(*) FROM cv_types WHERE ctid <> '(0,1)' GROUP BY grp P11 Predicates EXISTS over an array of objects (same-element semantics), @> containment, jsonb_path_exists / @? Yes sift_col, sift_table SELECT count(*) FROM orders o WHERE EXISTS (SELECT 1 FROM jsonb_to_recordset(o.items) i(category text, qty int) WHERE i.category = 'Toys' AND i.qty > 2) Several tests of one element and AND/OR/NOT of tests; also in FILTER; a jsonb attribute @> an object whose members are paths of the index (scalars, nested objects, arrays). fixture:exists_same_element,fixture:contains,fixture:path_exists agg @exists_same_element P12 Predicates jsonpath with !=, !(), a null literal, constant variables and strict mode (without !); @> with a JSON null member Yes the literal has the JSON type of the member's leaf (text, integer, boolean); complete kinds (minor 6) enable pushdown; old units with uncertain kinds or values the leaf cannot hold keep the normal plan sift_col, sift_table WHERE jsonb_path_exists(items, '$[*] ? (@.kind != "gift")') A JSON null differs from every literal but null; an absent key or a scalar element compares false, so true under !(). The kinds of the leaf tell a JSON null from an absent key. Truth tables: docs/AGG.md. REINDEX or VACUUM FULL restores complete kinds from the original documents. fixture:t_path_not_equal,fixture:t_path_negated_filter,fixture:t_path_variable,fixture:t_path_strict,fixture:t_contains_null,fixture:jp_ne_types,fixture:jp_null_literal,fixture:jp_not,fixture:jp_vars,fixture:jp_strict,test:agg_jsonpath_kinds agg @t_path_not_equal P17 Predicates The same tests with another literal type than the leaf's, a number against a float leaf, strict mode with !(), jsonb_path_match / @@, @> with a float, {} or a null among scalars No the heap result is not provable from the leaf: the members of another type are unknown, a float is stored rounded, the kinds of scalar elements are not kept sift_col, sift_table WHERE jsonb_path_exists(doc, '$[*] ? (@.n != "3")') The query keeps the normal plan. fixture:jp_ne_mismatch,fixture:jp_strict_not,fixture:jp_float_member,fixture:t_path_type_mismatch,fixture:t_path_float_literal,fixture:t_contains_float,fixture:t_contains_empty_object plain @jp_ne_mismatch P13 Predicates An element compared with a column of the outer row (= <> < <= > >=), in EXISTS and NOT EXISTS Partly a stored column of the table (not an expression of one) of the same type class as the member: an int4 member against a bigint column, not against a float8 one sift_col, sift_table WHERE EXISTS (SELECT 1 FROM jsonb_to_recordset(o.items) i(qty int) WHERE i.qty = o.tenant_id) The element vectors are compared with the value of the column of each element's row; NULL on either side is unknown, not true. Integer and float members (a NaN equals itself and is the greatest value); the other tests of the same element can be around it. P20 has the shapes that keep the normal plan. fixture:eo_ops,fixture:eo_ops_commuted,fixture:eo_not_exists,fixture:eo_rows,fixture:eo_one_of_many,fixture:eo_widths,fixture:eo_float,fixture:eo_orders_customer,fixture:eo_outer_unnest,fixture:eo_outer_other_array,test:aggstub_exists agg @eo_ops P20 Predicates An element compared with an expression of the outer row or a column of another type class, a second array in the sub-query, LIMIT or OFFSET in it No the test is no comparison of an element leaf with a column leaf of the row sift_col, sift_table WHERE EXISTS (SELECT 1 FROM jsonb_to_recordset(o.items) i(qty int) WHERE i.qty = o.tenant_id % 5) fixture:t_exists_outer_reference,fixture:t_exists_offset,fixture:t_exists_two_arrays,fixture:eo_trap_expression,fixture:eo_trap_type,fixture:eo_trap_limit,fixture:eo_trap_offset,fixture:eo_trap_two_arrays,fixture:eo_trap_row_test,fixture:eo_trap_outer_element plain @t_exists_outer_reference P16 Predicates Comparison with a bound parameter in a prepared statement (custom or generic plan) Yes sift_col PREPARE p(date, date) AS SELECT ... WHERE created >= $1 AND created <= $2 GROUP BY k; EXECUTE p(...) The values are known when the scan starts. A generic plan guesses the selectivity of the parameter; the scan is not charged for that guess, as the normal plan reads every row whatever the value is. test:col_agg_generic agg -- pre: SET plan_cache_mode = force_generic_plan;\n-- pre: PREPARE gp(int[], date, date) AS SELECT o.country_code, count(DISTINCT o.id), sum(i.qty) FROM orders o CROSS JOIN LATERAL jsonb_to_recordset(o.items) AS i(qty int) WHERE o.tenant_id = ANY($1) AND o.placed_at >= $2 AND o.placed_at <= $3 GROUP BY 1;\nEXECUTE gp('{1,2,3,4,5,6,7,8,9,10}', '2024-01-01', '2026-12-31') P18 Predicates A set filter bound as one array parameter, col = ANY($1::int4[]), in a generic plan Yes sift_col PREPARE sa(int4[]) AS SELECT ... WHERE tenant_id = ANY($1) GROUP BY k; EXECUTE sa(...) The planner cannot see how long the array is (it assumes 10 elements): the index and bitmap scans of such a set are costed for sift.agg_param_set_size elements (default 1000), so the generic plan takes the scan. A set of about 100 members is faster on a custom plan. test:agg_set_array agg -- pre: SET plan_cache_mode = force_generic_plan;\n-- pre: PREPARE sa(int4[]) AS SELECT status, count(*) FROM orders WHERE tenant_id = ANY($1) GROUP BY 1;\nEXECUTE sa('{1,2,3,4,5}') P19 Predicates A set filter bound as one array parameter on a text, int2, int8, date, timestamp or timestamptz column, in a generic plan Yes sift_col PREPARE sat(text[]) AS SELECT ... WHERE source = ANY($1) GROUP BY k; EXECUTE sat(...) The same plan as P17. Integer, date, int8, timestamp and timestamptz leaves test the array natively (a byte table over its range, else a sorted array or a hash table); a text leaf is tested by PostgreSQL once per distinct value, through a hash table of the array. uuid and bytea columns cannot be indexed by sift_col. test:agg_set_array agg ~ source -- pre: SET plan_cache_mode = force_generic_plan;\n-- pre: PREPARE sat(text[]) AS SELECT status, count(*) FROM orders WHERE source = ANY($1) GROUP BY 1;\nEXECUTE sat('{web,app}') P22 Predicates A partial sift_col index (CREATE INDEX .. WHERE pred): it answers a query whose conditions imply the index predicate Yes a query that does not imply the predicate keeps the normal plan (the index holds only the rows that pass it) sift_col SELECT grp, count(*), sum(amount) FROM t_partial WHERE deleted_at IS NULL AND day >= '2026-01-20' GROUP BY grp The planner honours the index predicate as it does for a BM25 index: the rows outside it are not in the index, so a scan is offered only when the query proves it (a prepared statement does in a custom plan, not in a generic one). Rows that move in and out of the predicate by UPDATE are indexed or dropped as inserts and deletes. fixture:pi_partial_implied,test:agg_partial_index agg @pi_partial_implied P23 Predicates A lookup or a range of a declared key (key_cut): the units are cut by the key, whatever the order of the heap Yes sift_col CREATE INDEX ON t USING sift_col (k, g, v) WITH (key_cut = 'k'); SELECT g, count(*) FROM t WHERE k BETWEEN 100 AND 140 GROUP BY g Opt-in (column format minor 17, new in 3.4; a 3.2 or 3.3 library cannot read the index). Promotion, merge and REINDEX cut again by the key, so the units keep their range under churn and after pg_repack; a key value of 600 sits in 1 of 100 units where a superblock order reads 8 of 8. test:col_key_cut agg -- pre: DROP TABLE IF EXISTS cov_kc;\n-- pre: CREATE TABLE cov_kc (id int, k int, g int, v int) WITH (autovacuum_enabled = off);\n-- pre: INSERT INTO cov_kc SELECT i, (i * 7919) % 5000, i % 7, i % 100 FROM generate_series(1, 20000) i;\n-- pre: CREATE INDEX cov_kc_col ON cov_kc USING sift_col (id, k, g, v) WITH (key_cut = 'k');\n-- pre: VACUUM (FREEZE, ANALYZE) cov_kc;\nSELECT g, count(*), sum(v) FROM cov_kc WHERE k BETWEEN 100 AND 140 GROUP BY g P24 Predicates An equality, an IN list or a short range on a column of many distinct keys, and the keys of a lookup join, read through the point directory (point_columns, point_tid_columns) Yes sift_col CREATE INDEX ON t USING sift_col (k, v) WITH (point_tid_columns = 'k'); SELECT count(*), sum(v) FROM t WHERE k IN (17, 4242, 3) Opt-in (column format minor 14, 16). The aggregate scan asks the directory for the rows of the keys: a unit that holds none is not read, and every conjunct still runs on the listed rows (EXPLAIN ANALYZE: Units From Point Directory). No index growth beyond the directory, no write cost measured on the aggregate path. test:agg_point_keys agg -- pre: DROP TABLE IF EXISTS cov_pk;\n-- pre: CREATE TABLE cov_pk (id int, k int, v int, pad text DEFAULT repeat('x', 400)) WITH (autovacuum_enabled = off);\n-- pre: ALTER TABLE cov_pk ALTER COLUMN pad SET STORAGE PLAIN;\n-- pre: INSERT INTO cov_pk (id, k, v) SELECT i, (i * 7919) % 9000, i % 100 FROM generate_series(1, 30000) i;\n-- pre: CREATE INDEX cov_pk_col ON cov_pk USING sift_col (id, k, v) WITH (point_tid_columns = 'k');\n-- pre: VACUUM (FREEZE, ANALYZE) cov_pk;\nSELECT count(*), sum(v) FROM cov_pk WHERE k IN (17, 4242, 3, 8999, 123456) WL01 Predicates EXISTS over jsonb_to_recordset of an array that an index expression returns, nested jsonpath tests (`$.a[*] ? (@.b == "x")`, `$.a[*].b`) and NOT of such a test Yes sift_col SELECT count(*) FROM t_work w WHERE EXISTS (SELECT 1 FROM jsonb_to_recordset(coldiff.doc_norm(w.doc) -> 'parts') o(category text, sign text) WHERE o.category = 'catg_j' AND o.sign = 'negative') The array is a list leaf of the index expression; the members of one element are tested together. A text-valued (astext) leaf is compared with text only. NOT of a path test is NULL for a document that is no object: the scan tests the presence leaf of the expression (an index expression always has it; an index built before 4.2 keeps the normal plan). fixture:wl_exists_parts,fixture:wl_exists_parts_two,fixture:wl_path_parts,fixture:wl_path_parts_op,fixture:wl_path_parts_not_expr,test:agg_workload_shapes agg @wl_exists_parts WL02 Predicates A number or boolean literal against a text-valued (astext) leaf, a strict jsonpath No the leaf holds the text of its scalars: a literal of another JSON type is not decided there; the heap raises for a strict path sift_col WHERE jsonb_path_exists(coldiff.doc_norm(w.doc), '$.parts[*] ? (@.score == 7)') fixture:wl_path_parts_number_text,fixture:wl_path_parts_strict,test:agg_workload_shapes plain @wl_path_parts_number_text WL03 Predicates jsonb_exists / ? on an object, as a filter or a group key (CASE WHEN doc ? k) No the JSON type of the document root is not in the index: a string or an array also answers ? and the key leaf cannot tell sift_col SELECT count(*) FROM t_work r WHERE jsonb_exists(coldiff.doc_norm(r.doc), 'mapped') `jsonb_typeof(doc -> k) IS NOT NULL` is the exact form for an object and is pushed. ? on an array of strings is pushed (P11). test:agg_workload_shapes plain SELECT count(*) FROM t_work r WHERE jsonb_exists(coldiff.doc_norm(r.doc), 'mapped') WL04 Predicates A jsonb value cast to a number in a filter ((metrics -> k)::int = 9) No the cast raises for a value that is no number: the scan could not give the same error for the rows it skips sift_col SELECT count(*) FROM t_work r WHERE (r.metrics -> 'm_total')::int = 9 The text path, `(metrics ->> k)::numeric::int`, is pushed. script:coverage plain SELECT count(*) FROM t_work r WHERE (r.metrics -> 'm_total')::int = 9 J01 Joins and subqueries INNER JOIN to a small table on equality (lookup join, up to 4 tables, up to 3 key columns) Yes sift_col, sift_table SELECT s.kind, count(*), sum(o.total) FROM orders o JOIN tenants s ON s.id = o.tenant_id GROUP BY s.kind The joined table is read at executor start with the query's snapshot, hashed by the key, and its columns are keys, filters and aggregate arguments like leaves. Meant for small dimension tables. fixture:j_inner_filter,fixture:t_lookup_inner_dup,fixture:t_lookup_row_level agg SELECT s.kind, count(*), sum(o.total) FROM orders o JOIN tenants s ON s.id = o.tenant_id GROUP BY s.kind 06:`JOIN` a small table, `EXISTS`, `IN (SELECT ...)` J02 Joins and subqueries LEFT JOIN to a small table (unmatched rows give NULLs), filters in ON or WHERE on the joined columns Yes sift_col, sift_table SELECT s.kind, count(o.id) FROM orders o LEFT JOIN tenants s ON s.id = o.tenant_id GROUP BY s.kind A NULL key matches nothing; text compares as = does (deterministic collations). fixture:t_lookup_missing_rows,fixture:j_on_filter,fixture:j_where_left agg @j_where_left 06:`JOIN` a small table, `EXISTS`, `IN (SELECT ...)` J03 Joins and subqueries EXISTS (...) and IN (SELECT ...) (semi join) Yes sift_col, sift_table SELECT count(*) FROM orders o WHERE EXISTS (SELECT 1 FROM tenants s WHERE s.id = o.tenant_id AND s.kind = 'outlet') A row stays once however many rows match. fixture:j_semi_exists,fixture:j_semi_in_subquery agg @j_semi_exists 06:`JOIN` a small table, `EXISTS`, `IN (SELECT ...)` J04 Joins and subqueries NOT EXISTS (anti join) Yes sift_col, sift_table SELECT count(*) FROM orders o WHERE NOT EXISTS (SELECT 1 FROM tenants s WHERE s.id = o.tenant_id AND s.kind = 'outlet') fixture:j_anti_not_exists agg @j_anti_not_exists 06:`JOIN` a small table, `EXISTS`, `IN (SELECT ...)` J05 Joins and subqueries NOT IN (SELECT ...) Yes the sub-select is uncorrelated, hashed, read-only, volatile-free and cannot raise; runs without parallel workers sift_col SELECT count(*) FROM orders WHERE tenant_id NOT IN (SELECT id FROM tenants WHERE kind = 'outlet') NOT IN (SELECT ...) and IN (SELECT ...) under OR or NOT: the sub-select runs once, as in the normal plan, and the NULL rule is PostgreSQL's. A test of two columns, of an int8 column or inside a key of an outer join's nullable input asks the sub-plan for each tuple. A correlated sub-select stays plain: write NOT EXISTS. fixture:t_not_in_subquery,fixture:t_not_in_subquery_or,fixture:t_in_subquery_or,test:agg_subplan_lut agg @t_not_in_subquery 06:`JOIN` a small table, `EXISTS`, `IN (SELECT ...)` J06 Joins and subqueries RIGHT JOIN, FULL JOIN to a table Yes a table joined by one equality to a column of the indexed table, ON clause or not (a condition of its own never filters its rows: they stay); a WHERE the indexed table's NULL row passes and a second join stay plain; the preserved side is a table (partitioned too), a view or a sub-select with conditions, a VALUES list or a function (as J09); no parallel workers sift_col, sift_table SELECT s.kind, count(o.id) FROM orders o FULL JOIN tenants s ON s.id = o.tenant_id GROUP BY s.kind RIGHT JOIN, FULL JOIN and LEFT JOIN with the indexed table on the right aggregate over the table that keeps its rows: a row nothing matches is one row of NULLs in the groups, as in the normal plan (count(o.id) is 0, count(*) is 1). The table is read whole when the scan starts and its rows are marked as the scan matches them. fixture:ps_right,fixture:ps_full_stores,fixture:ps_full_customer,fixture:ps_full_status,fixture:ps_multi_key_full,fixture:ps_empty_fact_full,fixture:ps_trap_where_fact,fixture:ps_trap_two_joins,fixture:pa_join_right,fixture:pa_join_full,fixture:j_dim_partitioned_right,fixture:j_dim_partitioned_full,fixture:j_full_filtered,fixture:j_full_filtered_on,fixture:j_full_scanned_filtered,fixture:j_full_chain agg SELECT s.kind, count(o.id) FROM orders o FULL JOIN tenants s ON s.id = o.tenant_id GROUP BY s.kind 13:`RIGHT JOIN`, `SELECT DISTINCT`, volatile functions J07 Joins and subqueries The indexed table on the nullable side of an outer join Yes as for J06: the only join, and no WHERE the indexed table's NULL row passes sift_col, sift_table SELECT s.kind, count(o.id) FROM tenants s LEFT JOIN orders o ON o.tenant_id = s.id GROUP BY s.kind LEFT JOIN from a dimension table to the indexed one aggregates the same way: the dimension rows with no order are groups too. A spine of spans joined by a range (`items.d <@ spine.range`) has no lookup key: every tuple is tested against every row of the spine, and the planner takes the cheaper plan; for a large spine write the aggregate first and join the spine above it. fixture:ps_left_stores,fixture:ps_left_kinds,fixture:ps_left_on_dim,fixture:ps_left_where_dim,fixture:ps_left_having,fixture:ps_dup_source,fixture:ps_empty_dim,fixture:ps_null_dim_keys,fixture:ps_left_values,fixture:ps_full_subselect,fixture:ps_range_on,fixture:pt_spine_left,test:agg_plan_tables agg SELECT s.kind, count(o.id) FROM tenants s LEFT JOIN orders o ON o.tenant_id = s.id GROUP BY s.kind J08 Joins and subqueries Join to a large table, self join Partly the joined table is read whole at executor start; fine for dimension tables, wasteful for big ones sift_col SELECT count(*) FROM orders a JOIN orders b ON a.customer_id = b.customer_id AND a.id < b.id The rows of a sub-select, VALUES list or function are bounded by memory (`sift.agg_lookup_max_kb`), not by a row count; the rows of a joined table that was loaded are shared with the other backends (`sift.agg_lookup_shared_kb`, on by default), so many sessions that join the same dimension load it once. script:coverage agg SELECT count(*) FROM orders a JOIN orders b ON a.customer_id = b.customer_id AND a.id < b.id J09 Joins and subqueries Join with no condition (ON true), join to a sub-select with GROUP BY or LIMIT, VALUES, a function Partly the right side is not volatile, has an equality with the table, has rows that fit sift.agg_lookup_max_kb (estimated), and is not on the side an outer join keeps whole; no workers sift_col SELECT s.kind, count(*) FROM orders o JOIN (SELECT id, kind FROM tenants LIMIT 5) s ON o.tenant_id = s.id GROUP BY s.kind A VALUES list, a sub-select PostgreSQL keeps (GROUP BY, LIMIT, DISTINCT, a set operation) or a function (generate_series) is run by its own plan once when the scan starts, and its rows are the lookup table, so duplicate and NULL keys join as in the normal plan (a sub-select with LIMIT and no ORDER BY gives the rows its plan gives, as the normal plan does). ON true (not lateral), random() inside the right side, and a table on the nullable side of the join stay on the normal plan (a condition PostgreSQL pushed into the sub-select is enforced by its plan). fixture:pt_values_inner,fixture:pt_values_left,fixture:pt_values_int,fixture:pt_subq_group,fixture:pt_subq_limit,fixture:pt_subq_self,fixture:pt_series,fixture:pt_keytypes,fixture:pt_volatile,fixture:pt_lateral,fixture:pt_on_true,fixture:pt_cap,test:agg_plan_tables agg ~ Lookup Joins: s \(subquery\) @pt_subq_limit J10 Joins and subqueries CTE referenced once, subquery in FROM (pulled up), view Yes sift_col, sift_table WITH x AS (SELECT * FROM orders WHERE tenant_id < 50) SELECT status, count(*) FROM x GROUP BY status fixture:t_lookup_case agg WITH x AS (SELECT * FROM orders WHERE tenant_id < 50) SELECT status, count(*) FROM x GROUP BY status J11 Joins and subqueries CTE referenced more than once (several granularities of one series) Partly planned inline when that plan has a SiftAgg and costs half; not for a CTE with a volatile function sift_col WITH base AS (...) SELECT ... FROM base GROUP BY 1 UNION ALL SELECT ... FROM base GROUP BY 2 The arms of one inlined CTE run in one pass over the index. fixture:c_shared_cte_series,fixture:c_shared_cte_unmatched,fixture:c_shared_cte_volatile agg WITH base AS (SELECT o.id AS order_id, o.placed_at, i.qty FROM orders o CROSS JOIN LATERAL jsonb_to_recordset(o.items) AS i(qty int) WHERE o.deleted_at IS NULL) SELECT 'day' AS grain, placed_at AS bucket, count(DISTINCT order_id) AS orders, sum(qty) AS units FROM base GROUP BY 2 UNION ALL SELECT 'month', date_trunc('month', placed_at)::date, count(DISTINCT order_id), sum(qty) FROM base GROUP BY 2 J12 Joins and subqueries UNION ALL of aggregates over the table Yes sift_col, sift_table SELECT 'a', status, count(*) FROM orders GROUP BY status UNION ALL SELECT 'b', lang, count(*) FROM orders GROUP BY lang Each arm is its own scan unless they share a CTE. fixture:month_series agg SELECT 'a' AS g, status, count(*) FROM orders GROUP BY status UNION ALL SELECT 'b', lang, count(*) FROM orders GROUP BY lang J13 Joins and subqueries Per-order aggregates (SELECT id, keys, agg ... GROUP BY id, keys) under an outer GROUP BY: one arm, or a common table expression planned inline in each arm Yes sift_col, sift_table SELECT category, avg(avg_price) FROM (SELECT o.id, i.category, avg(i.price) avg_price FROM orders o, jsonb_to_recordset(o.items) i(category text, price float8) GROUP BY 1, 2) x GROUP BY 1 The scan keeps each order's inner aggregates and adds them to the outer groups: no group per order. max and min as the inner aggregate of an arm, and FILTER or CASE over a text leaf in one, keep the normal plan (the inner aggregates are count, sum and avg); a non-null constant as an arm's key too (NULL is pushed). fixture:per_order_one_arm,fixture:per_order_inlined agg ~ \(inner @per_order_one_arm J14 Joins and subqueries Per-order aggregates with the outer aggregate in the same scan: UNION ALL arms (up to 4, one union may be inside another), HAVING or WHERE on the inner aggregates, gates, ROLLUP / GROUPING SETS, count(DISTINCT id) Yes sift_col, sift_table SELECT category, kind, avg(avg_price), count(DISTINCT order_id) FROM (... GROUP BY order_id, ...) x GROUP BY ROLLUP (category, kind) The scan keeps each order's inner aggregates and adds them to the outer groups: no group per order. fixture:per_order_union,fixture:per_order_gate,fixture:per_order_union_sets,fixture:t_two_level_where_exprs,fixture:per_order_nested_union agg ~ \(inner @per_order_union arms-outer J15 Joins and subqueries Per-order aggregates whose arms group by more keys than the outer query, and count(DISTINCT id) with no outer key Yes sift_col, sift_table SELECT cc, count(DISTINCT order_id), sum(units) FROM (... GROUP BY order_id, cc, category UNION ALL ... GROUP BY order_id, cc) x GROUP BY cc fixture:per_order_hidden_country,fixture:per_order_global_distinct agg ~ Cell Keys -- key: -\nWITH base AS (SELECT o.id AS order_id, o.country_code AS cc, i.category AS mc, i.qty FROM orders o CROSS JOIN LATERAL jsonb_to_recordset(o.items) AS i(category text, kind text, qty int, price float8, discount float8) WHERE o.tenant_id < 120 AND o.deleted_at IS NULL) SELECT cc, count(DISTINCT order_id) AS orders, count(*) AS cells, sum(units) AS units FROM (SELECT order_id, cc, mc AS category, sum(qty) AS units FROM base GROUP BY order_id, cc, mc UNION ALL SELECT order_id, cc, NULL::text, sum(qty) FROM base GROUP BY order_id, cc) x GROUP BY cc ORDER BY cc NULLS FIRST arms-cells J16 Joins and subqueries Per-order form with a volatile or mixed outer qual, a qual across arms, or more than 8 cell keys No the qual is per outer group and cell, not per row sift_col WHERE avg_price > 100 OR kind = 'x' The inner arms may still run in the scan; up to 8 cell keys run in the outer aggregate too. Up to 16 UNION ALL arms are one scan (4 before); the 17th keeps the normal plan under the slug `shape_width`. fixture:t_two_level_where_volatile,fixture:t_two_level_where_mixed,fixture:t_two_level_not_per_row,fixture:t_two_level_hidden_9 ok @t_two_level_where_volatile J17 Joins and subqueries Key of a join as an expression of leaves (small.k = lower(o.source)) or ON conditions that are not keys Yes sift_col, sift_table SELECT m.derived_category, count(*) FROM orders o JOIN categories_map m ON m.category = lower(o.source) GROUP BY 1 fixture:j_key_expr,fixture:j_on_filter agg @j_key_expr J18 Joins and subqueries Lookup join with a volatile key or filter, a nondeterministic collation, more than 32 tables or 128 combos No the combo cache evaluates once per distinct tuple, PostgreSQL per row sift_col, sift_table ON small.k = lower(o.source) AND random() < 0.5 Up to 32 joined tables and 128 combos push down; one more of either keeps the normal plan, and the reason (`sift.explain_why`, slug `shape_width`) names the limit. Test `agg_shape_widths`. fixture:t_key_expr_volatile plain @t_key_expr_volatile J19 Joins and subqueries Two unnests in one query (jsonb_array_elements of two arrays) and other set-returning functions No one lateral unnest of an indexed array or object per query sift_col, sift_table FROM orders o, jsonb_array_elements(o.items) a, jsonb_array_elements(o.notes) n One unnest of an array of strings with jsonb_array_elements_text pushes down (path tags[] text). fixture:t_exists_two_arrays,fixture:t_srf_select_list,fixture:t_tags_elements_text plain SELECT o.status, count(*) FROM orders o, jsonb_array_elements(o.items) a, jsonb_array_elements(o.notes) n GROUP BY 1 J20 Joins and subqueries Partitioned tables, inheritance parents Partly every partition left after planning is a sift_table or has a sift_col index, all of one kind, of the same layout and the same options; an inheritance parent, a table split by hash on a column that a parameter is compared with other than by an equality of its type, a partition of another storage and a text search stay on the normal plan sift_col SELECT status, count(*) FROM orders_part GROUP BY status The partitions PostgreSQL keeps after pruning are one source: their units one after the other, each partition read through its own index, or through itself when it is a sift_table (the attribute numbers may differ; the columns of a sift_table partition in the same order), the aggregate, the groups and the parallel workers over all of them. The plan lists the partitions (`Partitions:`). The index is created on each partition or on the parent. A partition without the index, with other options, or an inheritance parent keeps the normal plan. A parameter of a generic plan or a stable function on the partition key prunes partitions when the scan starts (`Partitions Removed`). Privileges and row security are those of the partitioned table. fixture:t_partitioned,fixture:pa_group_count,fixture:pa_global,fixture:pa_prune_range,fixture:pa_prune_default,fixture:pa_key_part,fixture:pa_subpartitioned,fixture:pa_join,fixture:pa_trap_no_index,fixture:pa_trap_options,fixture:pa_trap_inherits,fixture:pa_trap_only,fixture:pa_prune_param,fixture:pa_prune_hash_param,fixture:pa_prune_stable,test:agg_partitions,fixture:pa_join_first,fixture:pa_join_first_left,fixture:pa_join_right,fixture:pa_join_full,fixture:j_dim_partitioned,test:tbl_partitions agg ~ Partitions: SELECT status, count(*) FROM orders_part GROUP BY status J21 Joins and subqueries Row-level security that applies to the role No the scan does not apply policies sift_col SELECT grp, count(*) FROM cv_rls GROUP BY grp Superusers and table owners bypass RLS and keep the scan. test:col_index_basic plain -- guc: role = sift_cov_user\nSELECT grp, count(*) FROM cv_rls GROUP BY grp J22 Joins and subqueries A lookup join with no equality: a date in the ranges of a small table (<@, or its lower and upper bounds), INNER, LEFT, EXISTS / NOT EXISTS Yes sift_col, sift_table SELECT p.label, count(*) FROM orders o JOIN spans p ON o.placed_at <@ p.range GROUP BY p.label The rest of the ON clause is evaluated for each distinct value of the leaves and each row of the table: a date in several ranges is in each group, in none it is no row (a LEFT JOIN keeps it with NULL columns). fixture:k_range_lookup,fixture:k_range_lookup_bounds,fixture:k_range_lookup_unnest,fixture:k_range_lookup_left,fixture:k_range_lookup_exists agg ~ Lookup Joins: spans @k_range_lookup J23 Joins and subqueries A set in a table: IN (SELECT ..), EXISTS, NOT EXISTS on an integer or date leaf, answered from a byte table over the keys Yes sift_col SELECT status, count(*) FROM orders WHERE tenant_id IN (SELECT id FROM tenants WHERE kind = 'outlet') GROUP BY status The keys of the table (its own conditions applied once per row) are one byte each over their range, 4 MB at most per scan (sift.agg_semi_lut_size); wider keys, a condition on another column of the row or a text key use the generic lookup join, with the same rows. An inner join that returns no column of the table (a dimension that only filters: `JOIN tenants s ON s.id = o.tenant_id WHERE s.kind = 'outlet'`) is the same test when no key of the kept rows repeats. fixture:j_semi_lut_int,fixture:j_semi_lut_null_leaf,fixture:j_semi_lut_date,fixture:j_semi_lut_leaf_filter,test:agg_semi_lut,test:agg_inner_semi agg ~ Lookup Joins: tenants( \(semi\))?, tenants \(anti\) @j_semi_lut_int J24 Joins and subqueries A lookup join loads only the rows of the joined table that its own conditions can match: a btree on the leading columns, else the filter while loading Yes sift_col SELECT status, count(*) FROM orders WHERE tenant_id IN (SELECT id FROM tenants WHERE kind = 'outlet') GROUP BY status Conditions of the form column = constant (or a parameter), column <, <=, >, >= constant (both bounds of a range, together) or column IN (list) when the statistics expect the list to keep a small share of the table, on the leading columns of a btree of the joined table become an index scan; the others are applied to each row as it is read, so the lookup table keeps only the rows that can match. sift.agg_lookup_pushdown = off loads the whole table. test:agg_lookup_load,test:agg_lookup_in_list agg ~ Lookup Joins: tenants SELECT status, count(*) FROM orders WHERE tenant_id IN (SELECT id FROM tenants WHERE kind = 'outlet') GROUP BY status J25 Joins and subqueries UNION ALL or UNION of sub-selects joined to the table Yes the arms read no column of the query's other tables and no volatile function; no workers sift_col SELECT s.kind, count(*) FROM orders o JOIN (SELECT id, kind FROM tenants UNION ALL SELECT id, kind FROM tenants) s ON o.tenant_id = s.id GROUP BY s.kind PostgreSQL's plan of the arms is the child, run once when the scan starts; a key in both arms multiplies the rows, UNION removes the duplicate rows first, and a LEFT JOIN keeps the rows without a match. test:agg_append_children agg ~ Lookup Joins: s \(subquery\) SELECT s.kind, count(*) FROM orders o JOIN (SELECT id, kind FROM tenants UNION ALL SELECT id, kind FROM tenants) s ON o.tenant_id = s.id GROUP BY s.kind J26 Joins and subqueries LEFT JOIN LATERAL / CROSS JOIN LATERAL sub-select (LIMIT 1, UNION arms), generate_series or unnest of a column of the joined row Yes the child reads columns the index holds, or a non-raising call (jsonb_exists or ?, ->>, jsonb_typeof) of a document column whose call is an index expression column; no expression that can raise (division, cast of text, sub-select) beside another join; no column of a query above the aggregate's own; no workers sift_col SELECT s.kind, count(*) FROM orders o LEFT JOIN LATERAL (SELECT kind FROM tenants s WHERE s.id = o.tenant_id LIMIT 1) s ON true GROUP BY s.kind PostgreSQL's parameterized plan of the right side runs once for each distinct value of the columns it reads, and its rows multiply the joined row (duplicates and NULL-extended rows as in the normal plan). A child that reads the whole document stays plain. test:agg_correlated_children,test:agg_correlated_fanout,test:agg_derived_correlation agg SELECT s.kind, count(*) FROM orders o LEFT JOIN LATERAL (SELECT kind FROM tenants s WHERE s.id = o.tenant_id LIMIT 1) s ON true GROUP BY s.kind J27 Joins and subqueries IN (SELECT ..) in a group key, predicate or aggregate argument of an outer join's nullable input; a row-valued NOT IN Yes the sub-select is uncorrelated, hashed, read-only, volatile-free and cannot raise; no workers sift_col SELECT count(*) FROM orders WHERE (tenant_id, customer_id) NOT IN (SELECT id, 7 FROM tenants WHERE id IS NOT NULL) The scan asks PostgreSQL's own SubPlan for each tuple that reaches the test, so the NULL rule of IN and NOT IN is PostgreSQL's. test:agg_nullable_input_keys,test:agg_subplan_lut,test:agg_semi_lut agg ~ SubPlan SELECT count(*) FROM orders WHERE (tenant_id, customer_id) NOT IN (SELECT id, 7 FROM tenants WHERE id IS NOT NULL) WL05 Joins and subqueries x = ANY (ARRAY (SELECT ...)) of an uncorrelated, stable sub-select that cannot raise Yes sift_col SELECT count(*) FROM t_work w WHERE w.dim_a_id = ANY (ARRAY(SELECT e.id FROM t_work_dim e WHERE e.type = 'cat_a')) The InitPlan runs once when the scan starts and its rows are a lookup table of the leaf (parallel workers: none, the leader runs it). fixture:wl_any_array_subselect,fixture:wl_any_array_subselect_where,test:agg_workload_shapes agg @wl_any_array_subselect WL06 Joins and subqueries x = ANY (ARRAY (SELECT ...)) of a correlated, volatile or raising sub-select No the sub-select reads the row, changes between calls or may raise for a value the scan would not ask sift_col WHERE w.dim_a_id = ANY (ARRAY(SELECT e.id FROM t_work_dim e WHERE e.id = w.id % 60)) fixture:wl_any_array_subselect_correlated,fixture:wl_any_array_subselect_volatile,fixture:wl_any_array_subselect_raise,test:agg_workload_shapes plain @wl_any_array_subselect_correlated WL07 Joins and subqueries Per-item aggregates over a text-valued leaf with the cast in the aggregate (avg(n::float8), sum(n::float8)), under an outer aggregate Yes sift_col SELECT k, avg(a), count(DISTINCT rid) FROM (SELECT rid, k, avg(CASE WHEN ... THEN score::float8 END) a FROM ... GROUP BY rid, k UNION ALL ...) x GROUP BY k One fused scan; the cast runs for every item the scan reads, so the arms share one WHERE and no inner FILTER; arms with a WHERE of their own run one by one. fixture:wl_fused_astext,fixture:wl_fused_astext_having,fixture:wl_fused_astext_where,fixture:wl_fused_astext_where_differs,fixture:wl_fused_astext_raise,test:agg_workload_shapes agg ~ \(inner @wl_fused_astext WL08 Joins and subqueries Per-item arms with a second count(DISTINCT) over another column of the row (entities next to items) Partly the arms run one by one (one SiftAgg each); the fused scan counts the distinct rows only sift_col SELECT k, avg(a), count(DISTINCT rid), count(DISTINCT pid) FROM (SELECT rid, pid, k, avg(n) a FROM b GROUP BY rid, pid, k UNION ALL SELECT rid, pid, NULL::text, avg(n) FROM b GROUP BY rid, pid) x GROUP BY k script:coverage agg WITH b AS (SELECT r.id rid, r.dim_a_id pid, i.category k, i.score n FROM t_work r, jsonb_to_recordset(coldiff.doc_norm(r.doc) -> 'parts') AS i(category text, score float8) WHERE r.deleted_at IS NULL) SELECT k, avg(a), count(DISTINCT rid), count(DISTINCT pid) FROM (SELECT rid, pid, k, avg(n) a FROM b GROUP BY rid, pid, k UNION ALL SELECT rid, pid, NULL::text, avg(n) FROM b GROUP BY rid, pid) x GROUP BY k WL09 Joins and subqueries A lookup join keyed by a column of another joined table (map.type = entities.type) next to the keys read from the unnested elements Yes lookup keys may read columns of earlier joined tables sift_col SELECT sm.derived_category, count(*) FROM t_work r JOIN t_work_dim p ON p.id = r.dim_a_id CROSS JOIN LATERAL jsonb_to_recordset(coldiff.doc_norm(r.doc) -> 'parts') AS i(category text, val text) LEFT JOIN t_work_map sm ON sm.entity_type = p.type AND sm.category = i.category AND sm.val = i.val GROUP BY 1 test:agg_lookup_dependencies agg SELECT sm.derived_category, count(*) FROM t_work r JOIN t_work_dim p ON p.id = r.dim_a_id CROSS JOIN LATERAL jsonb_to_recordset(coldiff.doc_norm(r.doc) -> 'parts') AS i(category text, val text) LEFT JOIN t_work_map sm ON sm.entity_type = p.type AND sm.category = i.category AND sm.val = i.val GROUP BY 1 W01 Window, order, limit HAVING Yes sift_col, sift_table SELECT tenant_id, count(*) FROM orders GROUP BY tenant_id HAVING count(*) > 3 HAVING, projection, ORDER BY and LIMIT stay in the plan above the scan. fixture:f_having agg @f_having 07:`HAVING`, `ORDER BY ... LIMIT n` of the groups W02 Window, order, limit ORDER BY and LIMIT of an aggregated result (top-N groups) Yes sift_col, sift_table SELECT tenant_id, count(*) c FROM orders GROUP BY tenant_id ORDER BY c DESC LIMIT 3 The aggregate is the scan; the sort and the limit are PostgreSQL's. fixture:store_topn agg SELECT tenant_id, count(*) c FROM orders GROUP BY tenant_id ORDER BY c DESC, tenant_id LIMIT 3 07:`HAVING`, `ORDER BY ... LIMIT n` of the groups W03 Window, order, limit Window function over an aggregated subquery Yes sift_col, sift_table SELECT status, c, rank() OVER (ORDER BY c DESC) FROM (SELECT status, count(*) c FROM orders GROUP BY status) x fixture:store_category_top3 agg SELECT status, c, rank() OVER (ORDER BY c DESC) FROM (SELECT status, count(*) c FROM orders GROUP BY status) x 07:`HAVING`, `ORDER BY ... LIMIT n` of the groups W04 Window, order, limit Window function over the aggregate of the same query level: rank() OVER (ORDER BY count(*)), running sum, lag, share of the total Yes sift_col SELECT status, rank() OVER (ORDER BY count(*) DESC) FROM orders GROUP BY status The window runs in PostgreSQL above the groups of SiftAgg (sorted by the window, as for any aggregate plan). fixture:w_window_over_agg,fixture:w_window_month_lag,test:agg_window agg SELECT status, count(*), rank() OVER (ORDER BY count(*) DESC) FROM orders GROUP BY status W05 Window, order, limit Top-N rows: WHERE ... ORDER BY index columns [ASC|DESC] LIMIT n [OFFSET m] with no aggregate Partly the sort columns and the tie-break column (usually id) are in the sift_col index; the planner takes it when its cost is lower sift_col SELECT id, total FROM orders ORDER BY total DESC, id LIMIT 5 A bounded heap of the n + m best rows per scan, zone maps skip units, the winners are fetched from the heap by TID; EXPLAIN shows Top-N. A large n (about a third of the table) or a btree on the sort column can win by cost (sift.enable_topn = off removes the path). test:col_topn,fixture:h_topn_numeric,fixture:h_topn_two_keys agg ~ Top-N: total DESC SELECT id, total FROM orders ORDER BY total DESC, id LIMIT 5 08:`WHERE ... ORDER BY x LIMIT n` of rows W07 Window, order, limit Top-N rows of a text match: WHERE col @@> q [AND filters] ORDER BY created_at DESC LIMIT n Partly the BM25 index and a sift_col index with the sort columns are on the table; the planner takes it by cost against an ordered btree BM25 + sift_col SELECT id FROM docs WHERE body @@> 'wireless' ORDER BY created_at DESC, id LIMIT 10 The BM25 index gives the match set without reading the heap, the column index orders the matching rows; the heap is read n + m times, whatever the number of matches. Superblocks that cannot beat the heap are skipped (EXPLAIN: Superblocks Skipped By Top-N). test:col_topn,fixture:h_topn_search_date,fixture:h_topn_search_asc agg ~ Search: orders_fts.*Top-N: placed_at DESC SELECT id, total FROM orders WHERE memo @@> 'damaged' AND deleted_at IS NULL ORDER BY placed_at DESC, id LIMIT 10 W08 Window, order, limit Top-N sort keys: int2/4/8, bool, date, time, timestamp(tz), numeric(p<=18), money, float4/8, text, varchar, uuid, bytea, inet; ASC or DESC, NULLS FIRST or LAST, several keys, OFFSET Yes BM25 + sift_col SELECT id FROM t ORDER BY a DESC NULLS LAST, b, id LIMIT 7 OFFSET 3 Default btree order of the type; NaN is the largest. A text key compares the entries of the unit's dictionary with varstr_cmp under the collation of the query (byte order under C, strcoll then byte order under any deterministic collation), uuid, bytea and inet by the type's own comparison, so the rows are the normal plan's; such a key has no zones, so every unit that holds matching rows is read (the planner prices that). More than 4 sort keys sort every row that passes (W09). fixture:h_topn_numeric_nulls,fixture:h_topn_two_keys,fixture:h_topn_columns,fixture:topn_key_money,fixture:topn_key_money_asc,fixture:topn_key_time,fixture:topn_key_time_asc,fixture:topn_key_text,fixture:topn_key_text_c,fixture:topn_key_text_locale,fixture:topn_key_text_icu,fixture:topn_key_varchar,fixture:topn_key_uuid,fixture:topn_key_bytea,fixture:topn_key_inet,test:col_topn_text agg ~ Top-N: deleted_at DESC NULLS LAST SELECT id, deleted_at, total FROM orders ORDER BY deleted_at DESC NULLS LAST, total, id LIMIT 7 OFFSET 3 08:`WHERE ... ORDER BY x LIMIT n` of rows W09 Window, order, limit Top-N with a LIMIT or OFFSET that is a parameter, with n + m over 100000, or with more than 8 sort keys Yes sift_col SELECT id FROM t ORDER BY a, id LIMIT $1 A parameter of a generic plan is read when the scan starts: up to 100000 rows and 8 keys the best are kept in a heap, past that or with LIMIT NULL every row that passes is kept as keys and TID (an array in memory up to hash_mem, then a tuplesort at work_mem that spills) and sorted, and the heap is read only for the rows the Limit takes (not for the OFFSET rows when the output is plain columns). A constant n + m past 100000, more than 4 sort keys, or a heap that costs more than the sort is planned as that sort by cost. A single sort key with an order array holds the LIMIT only: an OFFSET past 100000 is walked in the array. test:col_topn,test:col_topn_deep,test:col_topn_deep_limits,fixture:topn_param_limit_10,fixture:topn_param_limit_cap1,fixture:topn_param_limit_null agg -- guc: plan_cache_mode = force_generic_plan\n-- prepare: (bigint)\n-- args: 10\nSELECT id FROM orders ORDER BY total, id LIMIT $1 W10 Window, order, limit Top-N by an expression the scan does not evaluate: a volatile or stable expression (random(), now(), concat(), timestamptz plus an interval, the text of a date), a sub-select, a result of a type the scan does not hold as a key (an interval, an array, jsonb), a numeric leaf of 19 or more digits No the scan evaluates a sort expression with PostgreSQL's own expression machinery for every row that passes (arithmetic, casts, CASE, coalesce, NULLIF, GREATEST/LEAST, boolean tests, IN lists, immutable functions and operators of numbers, strings, dates, bytea and inet, W20; coalesce(column, constant), lower and upper, W18), so the walker admits only the node kinds whose evaluation it has proved against the normal plan, and the key is stored as an integer, a float, a numeric, a string, a uuid, a bytea or an inet. A volatile or stable expression is never run by the scan: its value depends on the call, not on the row; the rest is refused with the reason named in sift.explain_why (topn_shape) sift_col, sift_table SELECT id FROM t ORDER BY now() - ts, id LIMIT 10 fixture:h_topn_expr_out,fixture:topn_xc_sublink,fixture:topn_xc_case_stable,fixture:topn_xc_case_volatile,fixture:topn_xs_result_interval,fixture:topn_xs_result_array,fixture:topn_xs_result_jsonb,fixture:topn_xp_volatile,fixture:topn_xp_stable,fixture:topn_xp_tz_interval,fixture:topn_xp_text_cast_date,fixture:topn_xp_numeric_wide,test:col_topn_arith,test:col_topn_case,test:col_topn_func true -- guc: seq_page_cost = 10000\n-- guc: random_page_cost = 10000\nSELECT coverage.keeps_normal('SELECT id FROM orders ORDER BY total, id LIMIT 5', ARRAY['SELECT id FROM orders ORDER BY concat(source, status), id LIMIT 5', 'SELECT id FROM orders ORDER BY placed_at::text, id LIMIT 5', 'SELECT id FROM orders ORDER BY CASE WHEN total > (SELECT 1) THEN 1 ELSE 0 END, id LIMIT 5']) W11 Window, order, limit Top-N of rows over an inner join to a table (its columns may be in the result), or an IN / EXISTS sub-select Partly a semi join (IN, EXISTS), or an inner join on a unique integer column, on conditions of its own columns, with or without columns of the joined table in the result; an outer join, a join on a column that is not unique, NOT EXISTS and a comparison with a scalar sub-select stay on the normal plan sift_col SELECT o.id, s.name FROM orders o JOIN tenants s ON s.id = o.tenant_id WHERE s.kind = 'outlet' ORDER BY o.total, o.id LIMIT 5 The keys of the joined rows that pass their conditions are a set of the scan (key IN (keys)), which the top-N holds exactly; a column of the joined table is read from the loaded table by the key of each row of the result. test:col_topn_join,fixture:topn_semi_in,fixture:topn_semi_exists,fixture:topn_semi_two,fixture:topn_semi_none,fixture:topn_semi_anti,fixture:topn_semi_text_key,fixture:topn_part_semi agg ~ Top-N: total, id.*Lookup Joins: tenants SELECT o.id, s.name FROM orders o JOIN tenants s ON s.id = o.tenant_id WHERE s.kind = 'outlet' ORDER BY o.total, o.id LIMIT 5 W15 Window, order, limit Top-N of rows with a window function that is not an aggregate over the whole set (row_number, rank, PARTITION BY, ORDER BY or a frame in the window) or FOR UPDATE, or over an outer join or a join whose other table may match a row twice No only a single table with no groups, windows or row marks, or an inner join on a unique key (W11); a SELECT DISTINCT is the group-by of K12 sift_col SELECT id, row_number() OVER (ORDER BY total) FROM orders ORDER BY total, id LIMIT 5 A join to a table without a unique index on its join column, or an outer join, keeps the normal plan. test:col_topn true -- guc: seq_page_cost = 10000\n-- guc: random_page_cost = 10000\nSELECT coverage.keeps_normal('SELECT id FROM orders ORDER BY total, id LIMIT 5', ARRAY['SELECT o.id, s.name FROM orders o LEFT JOIN tenants s ON s.id = o.tenant_id ORDER BY o.total, o.id LIMIT 5', 'SELECT o.id, m.tag FROM orders o JOIN map_dup m ON m.category = o.source ORDER BY o.total, o.id LIMIT 5', 'SELECT id, row_number() OVER (ORDER BY total) FROM orders ORDER BY total, id LIMIT 5', 'SELECT id FROM orders ORDER BY total, id LIMIT 5 FOR UPDATE']) W12 Window, order, limit Top-N of an OR of searches of one index: col @@> a OR col @@> b Partly every branch is a constant or parameter match of the same index, with no other predicate in the OR; with sift.score() or another predicate the normal plan stays BM25 + sift_col SELECT id FROM docs WHERE body @@> 'a' OR body @@> 'b' ORDER BY created_at DESC LIMIT 10 The match set is the union of the branches, each bound as its own query: its errors and its phrase rechecks are the normal plan's. A search ORed with a predicate on a column is S16. test:col_topn,fixture:topn_or_search_two,fixture:topn_or_search_phrase,fixture:topn_or_search_column agg SELECT id FROM orders WHERE memo @@> 'cable' OR memo @@> 'wire' ORDER BY placed_at DESC, id LIMIT 10 W13 Window, order, limit Top-N on a sift_table Yes sift_table SELECT id FROM t ORDER BY a, id LIMIT 10 The same top-N as on a sift_col index: the filters, the match set of a BM25 search and the numeric, date or integer sort columns are read from the column units and the write buffers, and the table only for the n rows of the result. A superblock is skipped when the zones of its units hold nothing better than the n-th row, whatever its place in the table: the new version of an updated row lands in the write buffer with any value. The limits of W09 to W12 apply. test:tbl_topn agg ~ Top-N: i4 SELECT id FROM cv_tt ORDER BY i4, id LIMIT 5 W17 Window, order, limit Top-N rows of a partitioned table Yes sift_col, sift_table SELECT id, total FROM orders_part ORDER BY total DESC, id LIMIT 5 The partitions after pruning are one scan: each keeps its units, the heap holds the n best rows of all of them with the partition of each, and the winners are fetched from the partitions' heaps (the attribute numbers of a partition may differ). A text search of the table stays on the normal plan (S13). fixture:topn_part_columns,fixture:topn_part_nulls,fixture:topn_part_pruned,fixture:topn_part_range,fixture:topn_part_sorted,fixture:topn_part_text,fixture:topn_part_ties agg ~ Top-N: total DESC, id SELECT id, total FROM orders_part ORDER BY total DESC, id LIMIT 5 W14 Window, order, limit Plain SELECT ... WHERE of many rows, window functions over rows No the column index serves aggregates and top-N, not row streams sift_col SELECT id, total FROM t WHERE total > 400 Use a btree index for row queries. script:coverage plain SELECT id, total FROM orders WHERE total > 400 W16 Window, order, limit Top-N by a text key under a nondeterministic collation, or by a char(n), name or cidr column No the equal strings of a nondeterministic collation differ as bytes and their order is the plan's; char(n), name and cidr have other comparison rules than their dictionary entries sift_col, sift_table SELECT id FROM t ORDER BY name COLLATE case_insensitive, id LIMIT 10 The scan keeps no heap row it cannot order exactly as the normal plan does, so these keys stay on the normal plan (`topn_shape` in `sift.explain_refusals`, with the reason). fixture:topn_key_text_nondet,test:col_topn_text true -- guc: seq_page_cost = 10000\n-- guc: random_page_cost = 10000\nSELECT coverage.keeps_normal('SELECT id FROM orders ORDER BY total, id LIMIT 5', ARRAY['SELECT id FROM orders ORDER BY status COLLATE coldiff_nd, id LIMIT 5', 'SELECT id FROM t_net ORDER BY cd, id LIMIT 5']) W18 Window, order, limit Top-N by coalesce(int, date or timestamp column, constant), lower(text column), upper(text column) Yes sift_col, sift_table SELECT id FROM t ORDER BY coalesce(a, 0) DESC, id LIMIT 10 These sort expressions cannot raise, so the scan that evaluates the key for every row that passes returns the normal plan's rows in its order. A NULL becomes the constant (the zone of a unit is widened to it, so units are skipped as for the column); lower() and upper() are PostgreSQL's own functions under the call's collation, called once per distinct entry of a unit. Any other expression is W10. fixture:topn_xf_coalesce_int,fixture:topn_xf_coalesce_int_desc,fixture:topn_xf_coalesce_int_low,fixture:topn_xf_coalesce_int_filter,fixture:topn_xf_coalesce_int4,fixture:topn_xf_coalesce_int2,fixture:topn_xf_coalesce_bigint,fixture:topn_xf_coalesce_date,fixture:topn_xf_coalesce_date_orders,fixture:topn_xf_coalesce_ts,fixture:topn_xf_coalesce_ts_orders,fixture:topn_xf_coalesce_tz,fixture:topn_xf_lower,fixture:topn_xf_lower_c,fixture:topn_xf_lower_locale,fixture:topn_xf_lower_varchar,fixture:topn_xf_lower_int,fixture:topn_xf_lower_filter,fixture:topn_xf_upper_desc,fixture:topn_xf_upper_locale,fixture:topn_xf_upper_long,test:col_topn_expr agg ~ Top-N: coalesce\(customer_id, 500\) DESC SELECT id, customer_id FROM orders ORDER BY coalesce(customer_id, 500) DESC, id LIMIT 7 08:`WHERE ... ORDER BY x LIMIT n` of rows W19 Window, order, limit Top-N page with the total of its set: aggregates over the whole set as window functions (count(*) OVER (), sum(x) OVER ()) Yes sift_col, sift_table SELECT id, total, count(*) OVER () FROM orders WHERE status = 'paid' ORDER BY total, id LIMIT 5 The scan keeps the heap of the n best and, in the same pass, computes the window aggregates from every row that passes (no unit is left out for the sort key's zone, and the scan is serial); every row of the page carries the totals. A window with PARTITION BY, ORDER BY or a frame that cuts the set, or a window function that is not an aggregate, keeps the normal plan (W15). An OFFSET past the last row returns no row, with or without the total. test:agg_topn_window agg ~ Pushed Aggregates: count\(\*\), sum\(orders\.total\) -- guc: seq_page_cost = 10000\n-- guc: random_page_cost = 10000\nSELECT id, total, count(*) OVER (), sum(total) OVER () FROM orders WHERE status = 'paid' ORDER BY total, id LIMIT 7 08:`WHERE ... ORDER BY x LIMIT n` of rows W20 Window, order, limit Top-N by an expression of the columns: arithmetic, casts, CASE, coalesce(a, b), NULLIF, GREATEST/LEAST, boolean tests, IN (..), lower(s || v), left(s, 3), md5(s), abs(a), round(n, 1), date_trunc('month', ts), a parameter of the statement Yes sift_col, sift_table SELECT id FROM t ORDER BY a + 1 DESC, id LIMIT 10 The scan evaluates the expression with PostgreSQL's own expression machinery (the backend's operators, casts and comparison) for every row that passes the WHERE, as the plan's sort does, so overflow (integer out of range), division by zero, NULL and NaN behave as in the normal plan and an error has the same SQLSTATE (the message may name another row). Only immutable expressions over the integer, float, numeric (18 digits or fewer), date, time, timestamp, text and uuid leaves of the index; a volatile or stable one stays with the normal plan. Zones skip units only for a plain column; the other expressions read every unit that holds matching rows, so the cost model charges them per row and keeps the normal plan where it would be slower (numeric, casts to text). Any immutable function or operator of the builtin catalog or the user's (the types between the leaves and the key do not matter: only the leaves and the result are stored), with a string, number, date, time, bytea, uuid or inet result. A parameter of a generic plan is read from the leader's executor state, so the scan runs in the leader; a function that is not parallel safe does the same. CASE and coalesce keep their lazy order (a branch that is not taken does not raise), NULLs follow three-valued logic, strings compare under the collation of the column. The rest is W10. fixture:topn_xp_add_i4,fixture:topn_xp_add_overflow,fixture:topn_xp_add_overflow_guard,fixture:topn_xp_add_overflow_late,fixture:topn_xp_sub_const,fixture:topn_xp_neg,fixture:topn_xp_neg_overflow,fixture:topn_xp_mul_i8,fixture:topn_xp_mul_i8_overflow,fixture:topn_xp_div_zero,fixture:topn_xp_div_guard,fixture:topn_xp_mod_ties,fixture:topn_xp_two_cols,fixture:topn_xp_two_cols_mul,fixture:topn_xp_multi_expr,fixture:topn_xp_expr_second,fixture:topn_xp_expr_first_mixed,fixture:topn_xp_filter,fixture:topn_xp_filter_empty,fixture:topn_xp_offset,fixture:topn_xp_float8,fixture:topn_xp_float_mixed,fixture:topn_xp_float_overflow,fixture:topn_xp_float_div,fixture:topn_xp_numeric,fixture:topn_xp_numeric_two,fixture:topn_xp_numeric_nan,fixture:topn_xp_cast_i4_i8,fixture:topn_xp_cast_i8_i4,fixture:topn_xp_cast_i8_i4_guard,fixture:topn_xp_cast_float,fixture:topn_xp_cast_numeric,fixture:topn_xp_cast_to_int,fixture:topn_xp_date_add,fixture:topn_xp_date_diff,fixture:topn_xp_ts_interval,fixture:topn_xp_ts_cast_date,fixture:topn_xp_text_cast,fixture:topn_xp_text_cast_c,fixture:topn_xp_text_cast_numeric,fixture:topn_xp_bool_cast,fixture:topn_xc_bool_and,fixture:topn_xc_bool_or_null,fixture:topn_xc_bool_test,fixture:topn_xc_case_date,fixture:topn_xc_case_desc,fixture:topn_xc_case_lazy,fixture:topn_xc_case_mixed_types,fixture:topn_xc_case_nested,fixture:topn_xc_case_raises,fixture:topn_xc_case_searched,fixture:topn_xc_case_simple,fixture:topn_xc_case_stable,fixture:topn_xc_case_text,fixture:topn_xc_case_text_cond,fixture:topn_xc_case_volatile,fixture:topn_xc_coalesce_arith,fixture:topn_xc_coalesce_cols,fixture:topn_xc_coalesce_float,fixture:topn_xc_coalesce_numeric,fixture:topn_xc_coalesce_numeric_cols,fixture:topn_xc_coalesce_text_cols,fixture:topn_xc_coalesce_three,fixture:topn_xc_coalesce_ts,fixture:topn_xc_coalesce_wide_const,fixture:topn_xc_greatest,fixture:topn_xc_greatest_text,fixture:topn_xc_in_list,fixture:topn_xc_in_list_text,fixture:topn_xc_least_date,fixture:topn_xc_nullif,fixture:topn_xc_nullif_text,fixture:topn_xc_string_compare,fixture:topn_xc_string_length,fixture:topn_xc_string_position,fixture:topn_xc_sublink,fixture:topn_xs_abs_int,fixture:topn_xs_abs_int_guard,fixture:topn_xs_bytea_encode,fixture:topn_xs_bytea_length,fixture:topn_xs_case_func_text,fixture:topn_xs_coalesce_func_text,fixture:topn_xs_concat_c,fixture:topn_xs_concat_lower,fixture:topn_xs_concat_sep,fixture:topn_xs_concat_upper_desc,fixture:topn_xs_date_trunc,fixture:topn_xs_epoch_diff,fixture:topn_xs_extract_date,fixture:topn_xs_floor_ceil,fixture:topn_xs_inet_abbrev,fixture:topn_xs_inet_funcs,fixture:topn_xs_initcap,fixture:topn_xs_left,fixture:topn_xs_ln_guard,fixture:topn_xs_ln_raises,fixture:topn_xs_lpad_id,fixture:topn_xs_make_date,fixture:topn_xs_md5,fixture:topn_xs_mod_func,fixture:topn_xs_nested_lower_upper,fixture:topn_xs_nested_upper_lower,fixture:topn_xs_param_case,fixture:topn_xs_param_coalesce,fixture:topn_xs_param_overflow,fixture:topn_xs_param_plus,fixture:topn_xs_param_text,fixture:topn_xs_power_overflow,fixture:topn_xs_repeat_left,fixture:topn_xs_replace,fixture:topn_xs_result_array,fixture:topn_xs_result_interval,fixture:topn_xs_result_jsonb,fixture:topn_xs_reverse,fixture:topn_xs_round_numeric,fixture:topn_xs_sqrt_abs,fixture:topn_xs_substr,fixture:topn_xs_time_plus,fixture:topn_xs_translate,fixture:topn_xs_trim_right,fixture:topn_xs_ts_interval_column,fixture:topn_xs_uuid_text,test:col_topn_arith,test:col_topn_case,test:col_topn_func agg ~ Top-N: \(customer_id \+ 1\) DESC SELECT id, customer_id FROM orders ORDER BY customer_id + 1 DESC, id LIMIT 7 08:`WHERE ... ORDER BY x LIMIT n` of rows W06 Window, order, limit FOR UPDATE, TABLESAMPLE on the indexed heap table, a set-returning function in the select list of the aggregate No row locks, sampling and per-row expansion are not aggregates over the index sift_col, sift_table SELECT ... FOR UPDATE fixture:t_for_update,fixture:t_tablesample plain @t_tablesample W21 Window, order, limit The first N rows of each group: row_number() OVER (PARTITION BY k ORDER BY o) cut by rn <= N (N > 1) Yes the planner costs the grouped scan below the window; a small work_mem or a table without a gain keeps the window (sift.explain_why: window_first) sift_col SELECT * FROM (SELECT grp, id, ts, row_number() OVER (PARTITION BY grp ORDER BY ts, id) rn FROM events) s WHERE rn <= 3 Planner only: the groups' arrays of each column, ordered, cut inside the group at N (up to 1,000) and unnested with their ordinal as rn; the scan keeps only the N best rows of each group as the rows pass (array_agg(x ORDER BY o)[1:N] in EXPLAIN). A nullable or several partition columns are the same form. rank() and dense_rank() <= N (up to 1,000) keep the rows that tie (the window function and its condition run above the rows the scan kept). N = 1 of row_number() is the DISTINCT ON rewrite. A table without a column index, a sub-select, an array or row typed column, or a window that is not this shape keeps the window. test:agg_topn_rownumber,test:agg_latest_per_group agg -- pre: DROP TABLE IF EXISTS cov_ev;\n-- pre: CREATE TABLE cov_ev (id int PRIMARY KEY, grp int NOT NULL, ts int, amount int NOT NULL) WITH (autovacuum_enabled = off);\n-- pre: INSERT INTO cov_ev SELECT i, i % 37, (i * 31) % 1000, i % 11 FROM generate_series(1, 5000) i;\n-- pre: CREATE INDEX cov_ev_col ON cov_ev USING sift_col (id, grp, ts, amount);\n-- pre: VACUUM (FREEZE, ANALYZE) cov_ev;\n-- guc: seq_page_cost = 1000\n-- guc: random_page_cost = 1000\nSELECT grp, id, rn FROM (SELECT grp, id, ts, row_number() OVER (PARTITION BY grp ORDER BY ts, id) rn FROM cov_ev) s WHERE rn <= 3 W22 Window, order, limit ORDER BY a column LIMIT n over units that are clean and all visible, from the head of each sorted run (order_columns) Yes sift_col CREATE INDEX ON t USING sift_col (id, k) WITH (order_columns = 'id, k'); SELECT id, k FROM t ORDER BY k, id LIMIT 20 Opt-in (column format minor 13, run heads minor 18). A unit keeps its first and last n / 128 keys (at most 32) with the heap addresses of their rows, so a LIMIT of up to 32 over clean units names its rows without reading a unit; a unit with a dead row or rows in the write buffer is read as before. test:col_run_heads,test:col_sorted_run agg -- pre: DROP TABLE IF EXISTS cov_rh;\n-- pre: CREATE TABLE cov_rh (id int, k int, pad text DEFAULT repeat('x', 300)) WITH (autovacuum_enabled = off);\n-- pre: ALTER TABLE cov_rh ALTER COLUMN pad SET STORAGE PLAIN;\n-- pre: INSERT INTO cov_rh (id, k) SELECT i, (hashint4(i) & 2147483647) % 500 FROM generate_series(1, 20000) i;\n-- pre: CREATE INDEX cov_rh_col ON cov_rh USING sift_col (id, k) WITH (order_columns = 'id, k');\n-- pre: VACUUM (FREEZE, DISABLE_PAGE_SKIPPING, ANALYZE) cov_rh;\n-- guc: seq_page_cost = 1000\n-- guc: random_page_cost = 1000\nSELECT id, k FROM cov_rh ORDER BY k, id LIMIT 20 X01 Parallel and workers Parallel scan of the aggregate (count, sum, avg, min, max, count(DISTINCT), percentiles, keys of any kind, lookup joins, grouping sets) Yes sift_col SELECT tenant_id, count(*), sum(total) FROM orders GROUP BY tenant_id Workers are the ones a parallel sequential scan of the table would get (table size, parallel_workers, max_parallel_workers_per_gather) and are costed: a scan under about 15 ms keeps the leader alone. Not under a Gather: Workers Planned in the SiftAgg node. The merge of the workers' tables is costed per group: a query with hundreds of groups keeps the leader alone. test:aggstub_scan,test:col_index_parallel agg ~ Workers Planned: [1-9] -- guc: max_parallel_workers_per_gather = 2\n-- guc: parallel_setup_cost = 0\n-- guc: parallel_tuple_cost = 0\n-- guc: min_parallel_table_scan_size = 0\nSELECT source, count(*), count(DISTINCT tenant_id) FROM orders GROUP BY source X02 Parallel and workers count(DISTINCT), sum(DISTINCT) and avg(DISTINCT) of numeric columns and numeric expressions (also dictionary and 32-bit integer values), percentiles, lookup-joined aggregates, per-order arms Yes sift_col SELECT source, count(DISTINCT total), sum(DISTINCT total) FROM orders GROUP BY source A worker sends the (group, value) pairs of its set, with the numbers themselves for a numeric value (the ids of its table of them are its own), and the leader gives them its own ids and counts or adds each once per group; a bitmap of 32-bit values goes as chunks that the leader ORs (X01). Proof: test `agg_distinct_parallel`. test:agg_distinct_parallel,test:aggstub_scan agg ~ Workers Planned: [1-9] -- guc: max_parallel_workers_per_gather = 2\n-- guc: parallel_setup_cost = 0\n-- guc: parallel_tuple_cost = 0\n-- guc: min_parallel_table_scan_size = 0\nSELECT source, count(DISTINCT total), sum(DISTINCT total) FROM orders GROUP BY source X03 Parallel and workers Float sums at any worker count Yes sift_col SELECT grp, sum(f8) FROM t GROUP BY grp 16 lanes added in lane order: the result is bit-identical for 0, 1, 2, ... workers and every SIMD level (not bit-identical to the heap-order sum of the normal plan). test:aggstub_scan true SELECT coverage.workers_agree('SELECT grp, sum(f), avg(f), count(*) FROM cv_dml GROUP BY grp') X04 Parallel and workers Generic aggregates with a combine function: stddev and variance, corr, regr_*, bit_and, bit_or, bit_xor, sum(float4) Yes SELECT source, stddev(total), bit_or(tenant_id) FROM orders GROUP BY source SELECT source, stddev(total), bool_and(total > 10) FROM orders GROUP BY source Each worker keeps the partial states per group and the leader merges them by key value with the aggregate's combine function. The ordered, DISTINCT and joined-column kinds stay serial (X06). bool_and and bool_or are the exception (X05). fixture:g_parallel,fixture:g_parallel_global,test:aggstub_generic agg ~ Workers Planned: [1-9] -- guc: max_parallel_workers_per_gather = 2\n-- guc: parallel_setup_cost = 0\n-- guc: parallel_tuple_cost = 0\n-- guc: min_parallel_table_scan_size = 0\nSELECT source, stddev_samp(total), bit_or(tenant_id), regr_count(total::float8, tenant_id::float8) FROM orders GROUP BY source aggs-parallel X05 Parallel and workers bool_and, bool_or, every in a parallel plan Yes sift_col, sift_table SELECT source, bool_and(tenant_id > 3) FROM orders GROUP BY source The partial states of the workers are merged by the aggregates' own combine functions. fixture:x_bool_parallel,fixture:x_bool_keys,fixture:x_bool_global,fixture:g_parallel agg ~ Workers Planned: [1-9] -- guc: max_parallel_workers_per_gather = 2\n-- guc: parallel_setup_cost = 0\n-- guc: parallel_tuple_cost = 0\n-- guc: min_parallel_table_scan_size = 0\nSELECT source, bool_and(tenant_id > 3), bool_or(total > 400) FROM orders GROUP BY source X06 Parallel and workers DISTINCT in a generic aggregate (string_agg, array_agg), count(DISTINCT) of an expression or of a 64-bit leaf through a combination of values, sum(DISTINCT) of an integer, a lookup join on a value, ordered sets of generic aggregates Partly run on the leader alone: no workers are planned (the reason is the aggregate: its partial state is not a set of numbers the leader can merge) sift_col SELECT source, string_agg(DISTINCT status, ',') FROM orders GROUP BY source The scan is still the column scan; only the parallelism is missing. test:aggstub_scan,test:agg_distinct_parallel agg !~ Workers Planned -- guc: max_parallel_workers_per_gather = 2\n-- guc: parallel_setup_cost = 0\n-- guc: parallel_tuple_cost = 0\n-- guc: min_parallel_table_scan_size = 0\nSELECT source, string_agg(DISTINCT status, ',') FROM orders GROUP BY source AP05 Pre-aggregates Shapes the pre-aggregates do not answer: two group keys of the columns, count(DISTINCT), float sums of the rows, a test on a column that is neither the grouped one nor zone-covered Partly SiftAgg scans the units instead of reading their metadata: the result is the same, the speed is the column scan's sift_col SELECT source, status, count(*) FROM orders GROUP BY source, status Only one key of the columns (dictionary text, int2/int4, date, bool), count, sum and avg of integers and numerics, and min/max (global, or per group of the one key: the cube keeps the minima and maxima) come from the stored histograms and sums; the leaves of the elements of an array or a map have the rollups of AP08 to AP10. Everything else decodes the leaves of the unit as before. test:agg_preagg true -- guc: seq_page_cost = 10000\n-- guc: random_page_cost = 10000\nSELECT coverage.scans('SELECT source, count(*), sum(total) FROM orders GROUP BY 1', ARRAY['SELECT source, status, count(*) FROM orders GROUP BY 1, 2', 'SELECT source, count(DISTINCT tenant_id) FROM orders GROUP BY 1', 'SELECT source, count(*) FROM orders WHERE status = ''paid'' GROUP BY 1']) preagg AP06 Pre-aggregates Pre-aggregates serve units whose pages are all visible, also with dead rows; after UPDATE or DELETE churn a unit is scanned until VACUUM writes the sums of its live rows (or a merge rewrites it) Partly the unit is clean (every heap page of its rows all-visible) and either at most sift.agg_dead_delta percent of its rows (default 15) are dead: counts and integer and numeric sums then come from the stored sums minus the dead rows, or the VACUUM that cleared them wrote the sums of its live rows (any dead share, every aggregate of the cube; not a unit with exceptions, overflow values, an element leaf or an inline string). VACUUM rewrites a unit once its dead rows reach sift_col option dead_rewrite_pct (default 10; 100 never rewrites, the sums of the live rows keep the unit served); PostgreSQL's VACUUM skips the index when few tuples are dead, use VACUUM (INDEX_CLEANUP on) sift_col SELECT source, count(*), sum(total) FROM orders GROUP BY source The result never changes, only the speed: a unit that neither the subtraction nor the sums of its live rows cover is scanned in the same plan and added to the exact ones; a single segment is rewritten by VACUUM too, no REINDEX needed. Rows inserted later are other units, answered once clean. test:agg_preagg true -- pre: DROP TABLE IF EXISTS cv_pa;\n-- pre: CREATE TABLE cv_pa AS SELECT i AS id, i % 7 AS g, i AS v FROM generate_series(1, 5000) i;\n-- pre: CREATE INDEX cv_pa_col ON cv_pa USING sift_col (id, g, v);\n-- pre: VACUUM (FREEZE, ANALYZE) cv_pa;\n-- pre: CREATE TEMP TABLE cv_pa_obs AS SELECT 'clean'::text AS s, coverage.preagg_units('SELECT g, count(*), sum(v) FROM cv_pa GROUP BY g') AS n;\n-- pre: DELETE FROM cv_pa WHERE id % 3 = 0;\n-- pre: INSERT INTO cv_pa_obs SELECT 'churn', coverage.preagg_units('SELECT g, count(*), sum(v) FROM cv_pa GROUP BY g');\n-- pre: VACUUM cv_pa;\n-- pre: INSERT INTO cv_pa_obs SELECT 'vacuum', coverage.preagg_units('SELECT g, count(*), sum(v) FROM cv_pa GROUP BY g');\n-- pre: DROP TABLE IF EXISTS cv_pa1;\n-- pre: CREATE TABLE cv_pa1 AS SELECT i AS id, i % 7 AS g, i AS v FROM generate_series(1, 5000) i;\n-- pre: CREATE INDEX cv_pa1_col ON cv_pa1 USING sift_col (id, g, v);\n-- pre: VACUUM (FREEZE, ANALYZE) cv_pa1;\n-- pre: DELETE FROM cv_pa1 WHERE id = 7;\n-- pre: VACUUM (INDEX_CLEANUP on) cv_pa1;\n-- pre: INSERT INTO cv_pa_obs SELECT 'few', coverage.preagg_units('SELECT g, count(*), sum(v) FROM cv_pa1 GROUP BY g');\n-- pre: DROP TABLE IF EXISTS cv_pa100;\n-- pre: CREATE TABLE cv_pa100 AS SELECT i AS id, i % 7 AS g, i AS v FROM generate_series(1, 5000) i;\n-- pre: CREATE INDEX cv_pa100_col ON cv_pa100 USING sift_col (id, g, v) WITH (dead_rewrite_pct = 100);\n-- pre: VACUUM (FREEZE, ANALYZE) cv_pa100;\n-- pre: DELETE FROM cv_pa100 WHERE id % 3 = 0;\n-- pre: VACUUM cv_pa100;\n-- pre: INSERT INTO cv_pa_obs SELECT 'never', coverage.preagg_units('SELECT g, count(*), sum(v) FROM cv_pa100 GROUP BY g');\nSELECT (SELECT n FROM cv_pa_obs WHERE s = 'clean') > 0 AND (SELECT n FROM cv_pa_obs WHERE s = 'churn') = 0 AND (SELECT n FROM cv_pa_obs WHERE s = 'vacuum') > 0 AND (SELECT n FROM cv_pa_obs WHERE s = 'few') > 0 AND (SELECT n FROM cv_pa_obs WHERE s = 'never') > 0 AND coverage.same('SELECT g, count(*), sum(v), min(v) FROM cv_pa GROUP BY g') preagg AP07 Pre-aggregates Pre-aggregates on a sift_table Yes sift_table SELECT grp, count(*), sum(i4) FROM t GROUP BY grp A unit of a stored segment answers from the sums and histograms its sidecar holds when its superblock has no write buffer row and no tombstone (`Units From Pre-aggregates` in EXPLAIN ANALYZE); the other units are scanned in the same plan. The cube is the default one of the segment's columns (up to 256 groups and 3 measures per dimension). After UPDATE or DELETE the superblocks that hold a tombstone are scanned until VACUUM rewrites the segment (`sift.table_dead_rewrite_pct`, default 50); rows of the write buffer are scanned. `sift.agg_preagg = off` scans every unit. test:tbl_preagg true -- pre: DROP TABLE IF EXISTS cv_pt;\n-- pre: CREATE TABLE cv_pt (id int, g int, v int) USING sift_table;\n-- pre: INSERT INTO cv_pt SELECT i, i % 7, i FROM generate_series(1, 5000) i;\n-- pre: VACUUM cv_pt;\n-- pre: CREATE TEMP TABLE cv_pt_obs AS SELECT 'clean'::text AS s, coverage.preagg_units('SELECT g, count(*), sum(v) FROM cv_pt GROUP BY g') AS n;\n-- pre: DELETE FROM cv_pt WHERE id % 3 = 0;\n-- pre: INSERT INTO cv_pt_obs SELECT 'churn', coverage.preagg_units('SELECT g, count(*), sum(v) FROM cv_pt GROUP BY g');\nSELECT (SELECT n FROM cv_pt_obs WHERE s = 'clean') > 0 AND (SELECT n FROM cv_pt_obs WHERE s = 'churn') = 0 AND coverage.same('SELECT g, count(*), sum(v) FROM cv_pt GROUP BY g') AP08 Pre-aggregates GROUP BY one leaf of the elements of an array (jsonb_to_recordset, jsonb_array_elements, an array of strings): count(*), count(m), sum, avg, min and max of the integer and float8 leaves of the same array Yes sift_col index of format minor 4 (3.1); a minor 3 segment has no element rollups sift_col SELECT i.category, count(*), sum(i.qty), avg(i.price) FROM orders o CROSS JOIN LATERAL jsonb_to_recordset(o.items) AS i(category text, qty int, price float8) GROUP BY i.category Answered from the rollup of the elements of the exact units: the distinct tuples of the low-cardinality leaves of the array with their counts and the count, sum, min and max of its numeric leaves, written with the unit (the default takes the leaves of at most a 32nd of the unit's elements; `rollup = 'items[].category, items[].kind'` names the dims, AP11). A test on the grouped leaf (=, IN, IS NULL, LIKE) runs on the tuples, a comparison on a column of the rows is a zone test as for AP04. A unit whose array overflowed, or that has no rollup with the dims and measures the query reads, is scanned and asks for a refresh (AP12). Float sums add the tuples in another order than the rows: the last digits of a float8 sum may differ. A test on a leaf that is not a dim, count(DISTINCT) and an expression aggregate scan. fixture:pa_elem_group,fixture:pa_elem_int,fixture:pa_elem_test,fixture:pa_roll_subset,test:agg_preagg_elem,test:agg_preagg_elem_parallel agg SELECT i.kind, count(*), sum(i.qty), avg(i.price) FROM orders o CROSS JOIN LATERAL jsonb_to_recordset(o.items) AS i(kind text, qty int, price float8) GROUP BY i.kind 09:pre-aggregated `count`, `sum`, `avg`, `min`, `max` AP09 Pre-aggregates GROUP BY two or more leaves of the elements of an array, in any order or a subset of them, with tests on them (=, <>, IN, IS NULL, LIKE, OR): count, sum, avg, min, max of the numeric leaves Yes sift_col index of format minor 4; units written with those leaves as dims (default: up to a 32nd of the elements as tuples, the dims of most distinct values dropped first; or the option rollup, AP11) sift_col SELECT i.category, i.kind, count(*), sum(i.qty), min(i.price), max(i.price) FROM orders o CROSS JOIN LATERAL jsonb_to_recordset(o.items) AS i(category text, kind text, qty int, price float8) WHERE i.kind IN ('standard', 'gift') OR i.category = 'Toys' GROUP BY i.category, i.kind An aggregate with FILTER (WHERE ...) on dims is evaluated on the tuples (a tuple whose elements fail the filter adds nothing to that aggregate, and still makes its group). A group key that is a function of one dim (`upper(kind)`, `left(day, 7)`, a `CASE` or a division of an integer dim) is evaluated once per distinct value of the tuples, as for a column. Grouping sets (GROUPING SETS, ROLLUP, CUBE) over the dims fold the tuples once per set, the keys outside a set being NULL; GROUPING() keeps the scan. One rollup per unit and array answers every subset of its dims: the tuples that agree on the group key are added up, so there is no split order to choose and a query by one of the dims reads the same blob as one by all of them. A unit that lacks a dim of the query scans; the others answer in the same plan. count(DISTINCT), percentile_*, a test on a leaf that is not a dim and a high-cardinality key (more than 4096 tuples in a unit) are not covered by design (docs/COL-PREAGG.md section 1). fixture:pa_roll_two_dims,fixture:pa_roll_three_dims,fixture:pa_roll_subset,fixture:pa_roll_filter_or,fixture:pa_roll_null_dim,fixture:pa_roll_lut_text,fixture:pa_roll_lut_int,fixture:pa_roll_filter_agg,fixture:pa_roll_sets,test:agg_preagg_rollup,test:agg_rollup_shapes agg SELECT i.category, i.kind, count(*), sum(i.qty), min(i.price), max(i.price) FROM orders o CROSS JOIN LATERAL jsonb_to_recordset(o.items) AS i(category text, kind text, qty int, price float8) WHERE i.kind IN ('standard', 'gift') OR i.category = 'Toys' GROUP BY i.category, i.kind 09:pre-aggregated `count`, `sum`, `avg`, `min`, `max` AP10 Pre-aggregates GROUP BY the key of a map (jsonb_each, jsonb_each_text): count, sum, avg, min, max of the value, tests on the key Yes sift_col index of format minor 4; the map declared as m.* in paths sift_col SELECT m.key, count(*), sum(m.value::float8), avg(m.value::float8), min(m.value::float8), max(m.value::float8) FROM orders o CROSS JOIN LATERAL jsonb_each_text(o.metrics) m WHERE m.key IN ('m_total', 'packaging') OR m.key LIKE 'check%' GROUP BY m.key The key of the map is the dim of the rollup of the map and its value the measure; the key ids of the unit are translated to the keys of the query as for a column. A test on the value, and a value cast other than to the declared type, scan. fixture:pa_roll_map,fixture:pa_roll_map_key_filter,test:agg_preagg_rollup agg SELECT m.key, count(*), sum(m.value::float8), avg(m.value::float8), min(m.value::float8), max(m.value::float8) FROM orders o CROSS JOIN LATERAL jsonb_each_text(o.metrics) m WHERE m.key IN ('m_total', 'packaging') OR m.key LIKE 'check%' GROUP BY m.key 09:pre-aggregated `count`, `sum`, `avg`, `min`, `max` AP11 Pre-aggregates CREATE INDEX or ALTER INDEX with a rollup option that names a leaf that is not a low-cardinality leaf of an array element or a map value declared in paths (or names it twice, or more than 6 for a list) Error 22023 at CREATE INDEX and ALTER INDEX; name integer, date, numeric, boolean or text leaves of an array of objects or a map declared in paths, or drop the option (rollup = 'off' disables the rollups) sift_col CREATE INDEX orders_roll_bad ON orders USING sift_col (id, items) WITH (paths = 'items[]{category text, qty int4}', rollup = 'items[].nosuch') WITH (rollup = 'items[].kind, items[].sub') names the dims of the rollups of the lists (the measures are their numeric leaves); the units written afterwards hold exactly these dims, up to 4096 tuples each and no byte budget, and the units written before are asked for on demand (AP12) or rewritten by a merge. The error names the leaf; a leaf with too many distinct values is not an error of the DDL: the unit holds no rollup for that list and scans. test:col_rollup_refresh error:22023 CREATE INDEX orders_roll_bad ON orders USING sift_col (id, items) WITH (paths = 'items[]{category text, qty int4}', rollup = 'items[].nosuch') AP12 Pre-aggregates A SELECT that scans exact units for lack of their rollup (none, other dims, a measure it lacks) asks for a refresh of their segments; a background worker rewrites them and the next SELECT answers from the rollups Yes sift.col_rollup_on_demand (on) and the index option rollup_on_demand (true); not on a hot standby; minor 3 segments are left as written sift_col SELECT (SELECT count(*) FROM sift.col_rollup_requests) >= 0 AND (SELECT requested >= refreshed FROM sift.col_rollup_state) The query's result does not change and it writes nothing (a read-only transaction asks too): it records the segment in a bounded queue in shared memory (EXPLAIN shows `Rollup Refresh Queued`); the worker rewrites a queued segment as a merge of it alone does, under the maintenance lock, so VACUUM, merges and writers are not blocked and a crash leaves the old segment. sift.col_rollup_state counts requested, refreshed and skipped; sift.col_rollup_requests lists the queue; sift.col_rollup_refresh(index) serves it in the session. test:col_rollup_refresh,test:col_rollup_ondemand true SELECT (SELECT count(*) FROM sift.col_rollup_requests) >= 0 AND (SELECT requested >= refreshed FROM sift.col_rollup_state) 09:pre-aggregated `count`, `sum`, `avg`, `min`, `max` AP13 Pre-aggregates After ALTER INDEX changes preagg, rollup or cube_minmax, the segments written before keep the old sections; sift.col_refresh(index) rewrites the ones that differ from what the options ask now, one segment at a time, with the same answers Yes sift.col_refresh(index, max_segments); a segment of a minor above sift.col_format_minor is left as written; not on a hot standby sift_col SELECT * FROM sift.col_refresh('orders_col', 0) Each segment is the atomic replacement of a merge of it alone, under the maintenance lock that readers and writers do not wait for; a cancel or a crash leaves the segments done and the one in hand as it was, and the call made again goes on. max_segments bounds the rewrites of a call (NULL: all, 0: only count what is stale). The planner counts the units that hold the histogram of a dim, its min and max, or the rollup of a list, and prices a scan answered from them by that share, so a half refreshed index costs between the two. Space freed is reused after VACUUM. test:col_refresh,test:col_refresh_crash,test:col_refresh_compat true SELECT count(*) = 1 FROM pg_proc WHERE proname = 'col_refresh' AND pronamespace = 'sift'::regnamespace AP14 Pre-aggregates Calendar cells of a date or timestamp beside jsonb paths: GROUP BY date_trunc(month, d) and date ranges of an index with paths Yes sift_col CREATE INDEX ON t USING sift_col (id, d, doc) WITH (paths = 'doc.a int4', calendar_columns = 'all'); SELECT date_trunc('month', d), count(*) FROM t GROUP BY 1 Opt-in with paths (3.4: calendar_columns = all or a list; 3.3 refused the combination): +9.3% to +12.1% of an index with three date columns. A range with day bounds cuts the month cells at its ends and answers the cells inside from the pre-aggregates; a date without a histogram is tested by its cells. test:agg_cal_paths agg -- pre: DROP TABLE IF EXISTS cov_cal;\n-- pre: CREATE TABLE cov_cal (id int, d date, doc jsonb, v int) WITH (autovacuum_enabled = off);\n-- pre: INSERT INTO cov_cal SELECT i, date '2024-01-01' + i % 365, jsonb_build_object('a', i % 5), i % 100 FROM generate_series(1, 20000) i;\n-- pre: CREATE INDEX cov_cal_col ON cov_cal USING sift_col (id, d, doc, v) WITH (paths = 'doc.a int4', calendar_columns = 'all');\n-- pre: VACUUM (FREEZE, ANALYZE) cov_cal;\nSELECT date_trunc('month', d) m, count(*), sum(v) FROM cov_cal WHERE d >= date '2024-03-15' AND d < date '2024-09-01' GROUP BY 1 AP15 Pre-aggregates A filter col IS NULL on a leaf that is NULL in nearly every row (a soft-delete stamp): a unit with a few rows where it is not NULL is answered from its pre-aggregates minus those exceptions Yes min and max of a group are scanned, not answered from the exceptions; a leaf that is not NULL-heavy keeps the scan of the unit sift_col SELECT kind, count(*), sum(val) FROM t WHERE stamp IS NULL GROUP BY kind Column format minor 19. A unit in which the leaf is NULL in nearly every row also keeps the cube and the element rollups of the few rows where it is not, with the dims and measures of its own; the scan subtracts them (EXPLAIN ANALYZE: Units Net Of Exceptions, Units With Exceptions Scanned). Before it, a stamp set in a few rows of every unit made every unit a scan. test:agg_preagg_exceptions true -- pre: DROP TABLE IF EXISTS cov_ex;\n-- pre: CREATE TABLE cov_ex (id int, kind int, val bigint, stamp timestamptz, pad text) WITH (autovacuum_enabled = off);\n-- pre: ALTER TABLE cov_ex ALTER COLUMN pad SET STORAGE PLAIN;\n-- pre: INSERT INTO cov_ex SELECT g, g % 6, (g % 997) * 3 - 1000, CASE WHEN g % 200 = 0 THEN timestamptz '2025-06-01 00:00:00+00' + (g % 50) * interval '1 day' END, repeat('p', 1500) FROM generate_series(1, 24000) g;\n-- pre: CREATE INDEX cov_ex_col ON cov_ex USING sift_col (id, kind, val, stamp);\n-- pre: VACUUM (FREEZE, ANALYZE, DISABLE_PAGE_SKIPPING) cov_ex;\nSELECT coverage.preagg_units('SELECT kind, count(*), sum(val) FROM cov_ex WHERE stamp IS NULL GROUP BY kind') > 0 D01 DML and visibility Rows inserted after the index was built, not yet vacuumed (pages not all-visible) Yes sift_col, sift_table SELECT grp, count(*), sum(v) FROM t GROUP BY grp Rows on pages the visibility map does not cover are checked in the heap under the query's snapshot, after the value filters. scenario:insert,test:col_index_dml agg SELECT grp, count(*), sum(v), avg(f) FROM cv_dml GROUP BY grp D02 DML and visibility Updated and deleted rows, rolled-back inserts (dead versions before VACUUM) Yes sift_col, sift_table SELECT tx, count(*) FROM t GROUP BY tx An UPDATE of an indexed column is a new tuple with its own index record. scenario:update,scenario:delete,scenario:abort agg SELECT tx, count(*), min(v), max(v) FROM cv_dml WHERE id % 25 <> 1 GROUP BY tx D03 DML and visibility The transaction's own uncommitted changes Yes sift_col, sift_table BEGIN; INSERT ...; SELECT count(*) ...; Compared with the normal plan in every state. test:tbl_agg,test:col_index_dml true SELECT coverage.in_tx(ARRAY['INSERT INTO cv_dml SELECT 20000 + i, i % 5, i, i, ''n'' FROM generate_series(1, 100) i', 'UPDATE cv_dml SET v = v + 1 WHERE id % 7 = 0', 'DELETE FROM cv_dml WHERE id % 11 = 0'], 'SELECT grp, count(*), sum(v), min(v) FROM cv_dml GROUP BY grp') AND coverage.in_tx(ARRAY['INSERT INTO cv_tt SELECT * FROM cv_types WHERE id <= 40', 'UPDATE cv_tt SET i4 = i4 + 1 WHERE id % 9 = 0', 'DELETE FROM cv_tt WHERE id % 13 = 0'], 'SELECT grp, count(*), sum(i4) FROM cv_tt GROUP BY grp') D04 DML and visibility Long snapshots, uncommitted writers, REPEATABLE READ and SERIALIZABLE Yes sift_col BEGIN ISOLATION LEVEL REPEATABLE READ; SELECT ... A SERIALIZABLE scan takes a relation predicate lock, so concurrent writers can cause 40001: retry. scenario:held,fixture:t_serializable true SELECT coverage.snapshot('repeatable read', 'SELECT grp, count(*), sum(v) FROM cv_snap GROUP BY grp', ARRAY['INSERT INTO cv_snap SELECT 5000 + i, i % 5, i FROM generate_series(1, 200) i', 'UPDATE cv_snap SET v = v + 1 WHERE id % 7 = 0', 'DELETE FROM cv_snap WHERE id % 13 = 0']) AND coverage.snapshot('serializable', 'SELECT grp, count(*), sum(v) FROM cv_snap GROUP BY grp', ARRAY['INSERT INTO cv_snap SELECT 5000 + i, i % 5, i FROM generate_series(1, 200) i', 'UPDATE cv_snap SET v = v + 1 WHERE id % 7 = 0', 'DELETE FROM cv_snap WHERE id % 13 = 0']) D05 DML and visibility VACUUM, REINDEX, promotion and merge of the segments, TID reuse Yes sift_col VACUUM orders; SELECT ... scenario:vacuum,scenario:maint,scenario:recycle true -- pre: DELETE FROM cv_maint WHERE id % 3 = 0;\n-- pre: VACUUM cv_maint;\n-- pre: INSERT INTO cv_maint SELECT 10000 + i, i % 5, i FROM generate_series(1, 700) i;\n-- pre: UPDATE cv_maint SET v = v + 1 WHERE id % 5 = 0;\n-- pre: SELECT sift.col_promote('cv_maint_col'), sift.col_merge('cv_maint_col');\n-- pre: REINDEX INDEX cv_maint_col;\n-- pre: DELETE FROM cv_maint WHERE id > 10000 AND id % 2 = 0;\n-- pre: VACUUM cv_maint;\n-- pre: INSERT INTO cv_maint SELECT 20000 + i, i % 5, i FROM generate_series(1, 300) i;\nSELECT coverage.same('SELECT grp, count(*), sum(v), max(id) FROM cv_maint GROUP BY grp') D06 DML and visibility HOT updates of columns no index holds Yes sift_col UPDATE orders SET total = total + 1 scenario:hot true -- pre: UPDATE cv_hot SET pad = pad || 'x' WHERE id % 2 = 0;\nSELECT EXISTS (SELECT 1 FROM heap_page_items(get_raw_page('cv_hot', 0)) WHERE (t_infomask2 & 16384) <> 0) AND coverage.same('SELECT grp, count(*), sum(v), max(id) FROM cv_hot GROUP BY grp') D07 DML and visibility Hot standby (physical replica) Yes sift_col, sift_table, BM25 SELECT ... on the standby The index is generic WAL: a standby serves the scans read-only. A logical subscriber builds its own index (proof: script:replication). script:replication external:standby SELECT grp, count(*), sum(v) FROM cv_dml GROUP BY grp S01 Search (BM25) All terms: body @@> 'cable wire' (words are ANDed) Yes BM25 SELECT id FROM docs WHERE body @@> 'battery friendly' Case and accent folded; optional Snowball English stemming (stemmer = 'en'). test:query_match,test:rank_scan scan SELECT id FROM orders WHERE memo @@> 'cable wire' S02 Search (BM25) OR, parentheses, set of words {a b c}, boost ^n Yes BM25 SELECT id FROM docs WHERE body @@> '(case OR clip) AND friendly^3' test:query_parse,test:query_match scan SELECT id FROM orders WHERE memo @@> '((cable OR wire) AND gift^3) OR {box label}' S03 Search (BM25) NOT (exclusion) Yes BM25 SELECT id FROM docs WHERE body @@> 'battery NOT squeaky' A NOT needs a positive term in its AND group. test:query_match scan SELECT id FROM orders WHERE memo @@> 'cable NOT wire' S04 Search (BM25) Phrase: "friendly staff" Yes BM25 SELECT id FROM docs WHERE body @@> '"friendly staff"' Decided from the word positions in the index (positions = on, the default); with positions = off, rechecked in the heap. test:phrase_positions scan SELECT id FROM orders WHERE memo @@> '"gift invoice"' S05 Search (BM25) Prefix (cle*), wildcard (w?fi), regex (/thank.*/), fuzzy (clen~1) Yes BM25 SELECT id FROM docs WHERE body @@> 'brea* OR n?isy OR /no.*y/ OR frendly~1' Above sift.max_expansions (512) terms a leaf scores as one term. A regex has no back-references. test:query_match,test:prefix_index scan SELECT id FROM orders WHERE memo @@> 'cle* OR w?fi OR /thank.*/ OR clen~1' S06 Search (BM25) Field weights: sift.field_query(query, 'title^3, body') over the sections of one key; name:word, name:(a b), name:{a b} Partly the index is built with the fields option on a key made by sift.sections(...) BM25 sift.sections(title, body) @@> sift.field_query('wireless', 'title^3, body') Weights are set per query and a row scores the sum of its fields. No BM25F, no analyzer per field, phrases and regular expressions are not scoped to a field ([FIELDS-FACETS.md](FIELDS-FACETS.md)). Without fields a colon is plain text. prefix_index = n makes short prefixes one lookup (autocomplete). test:fields,test:field_weights,test:prefix_index scan SELECT id FROM cv_places WHERE (name || chr(29) || admin1 || ' ' || country) @@> '(gr* ho*) AND (name:{gr* ho*}^3 OR (gr* ho*))' ORDER BY sift.score(ctid) DESC LIMIT 20 S07 Search (BM25) Ranked top-k: ORDER BY sift.score(ctid) DESC LIMIT k Yes BM25 SELECT id, sift.score(ctid) FROM docs WHERE body @@> 'case OR clip' ORDER BY sift.score(ctid) DESC LIMIT 10 Exact BM25; skips blocks that cannot reach the top k; ties break by ctid. Also with a prepared LIMIT $1 and on materialized views. test:rank_scan,test:rank_limit,test:rank_prune scan SELECT id, sift.score(ctid) FROM orders WHERE memo @@> 'cable OR wire' ORDER BY sift.score(ctid) DESC LIMIT 10 S08 Search (BM25) count(*) of a match Yes BM25 SELECT count(*) FROM docs WHERE body @@> 'friendly' Counted from the index (SiftCount). With another filter in the WHERE, a SiftScan feeds PostgreSQL's count. test:count_scan count SELECT count(*) FROM orders WHERE memo @@> 'cable wire' S09 Search (BM25) Match with other filters: score ranking plus a range or equality filter Yes BM25 SELECT id FROM docs WHERE body @@> 'friendly' AND published >= '2026-02-01' ORDER BY sift.score(ctid) DESC LIMIT 10 With sift.filter_pushdown (on) the bitmap of the other quals (btree, BRIN, GIN) goes into the ranked scan, so a selective filter stays fast; results equal the post-filter plan. Prepared statements keep the filter as a child built only when it pays. test:filter_pushdown,test:filter_pushdown_prepared scan SELECT id, sift.score(ctid) FROM orders WHERE memo @@> 'cable OR wire' AND placed_at >= DATE '2025-01-01' ORDER BY sift.score(ctid) DESC LIMIT 10 S10 Search (BM25) Join with ranking Yes BM25 SELECT d.id, c.name FROM docs d JOIN categories c ON c.id = d.category WHERE d.body @@> 'friendly' ORDER BY sift.score(d.ctid) DESC LIMIT 10 Nested loop; semi joins too. test:rank_join,test:rank_semi_join scan SELECT o.id, s.kind FROM orders o JOIN tenants s ON s.id = o.tenant_id WHERE o.memo @@> 'damaged' ORDER BY sift.score(o.ctid) DESC LIMIT 10 S11 Search (BM25) Several queries ORed: body @@> ANY (ARRAY['case', 'clip']) Yes scored like the OR in one query text; an unscored list above 100,000 rows stays a bitmap scan BM25 SELECT id FROM docs WHERE body @@> ANY (ARRAY['case', 'clip']) Same matches, scores and ranking as `a OR b`; count, LIMIT and top-N use the scan. Medians of 9 runs: count(*) 1.6-54x faster, LIMIT 1.5-130x faster (0.1 ms slower when the whole query takes under 0.2 ms), full list equal or up to 2x faster, plan under 0.6 ms at 100 elements. test:planner_array,test:array_any_score scan SELECT id, sift.score(ctid) FROM orders WHERE memo @@> ANY (ARRAY['cable', 'wire']) ORDER BY sift.score(ctid) DESC LIMIT 3 S12 Search (BM25) Keyword facets: sift.facets(index, facet_index, query, limit) Partly keyword analyzer, text keys, a table without row-level security BM25 SELECT * FROM sift.facets('docs_body', 'docs_category', 'friendly OR case', 10) Counts per value from the bitmaps, equal to the GROUP BY of the same snapshot; wins on broad queries over low-cardinality columns, GROUP BY wins on selective ones or thousands of values. test:facets ok SELECT count(*) FROM (SELECT value::text, count::bigint FROM sift.facets('orders_fts', 'orders_src_kw', 'cable OR wire', 100) EXCEPT ALL SELECT source, count(*) FROM orders WHERE memo @@> 'cable OR wire' GROUP BY source) x S13 Search (BM25) Partitioned tables: matching, count(*) and ranking through the parent Yes BM25 SELECT id FROM logs WHERE msg @@> 'disk "node 7"' ORDER BY sift.score(ctid) DESC LIMIT 3 One index per leaf partition, global top-k by a MergeAppend. Scores are per partition statistics; sift.score() needs an index on every partition read. test:rank_partition scan SELECT id, sift.score(ctid) FROM cv_logs WHERE msg @@> 'disk "node 7"' ORDER BY sift.score(ctid) DESC LIMIT 3 S14 Search (BM25) Several columns in one index Partly build one key with sift.sections(a, b) (or concatenate the columns) and query with the same expression BM25 CREATE INDEX ON docs USING sift (sift.sections(title, body)) WITH (fields = 'title,body') A multicolumn index is refused (next rows). test:fields,test:field_weights scan SELECT id FROM cv_places WHERE (name || chr(29) || admin1 || ' ' || country) @@> 'gr*' ORDER BY sift.score(ctid) DESC LIMIT 5 S15 Search (BM25) Match on a column with no sift index No no index: the operator runs per row, with the same results BM25 SELECT id FROM docs WHERE title @@> 'noisy' script:coverage noscan SELECT id FROM orders WHERE country_code @@> 'US' S16 Search (BM25) A match ORed with a predicate on another column Partly needs an index for the other branch; the planner takes BitmapOr when costs say so BM25 WHERE body @@> 'a' OR category = 1 script:coverage bitmap ~ BitmapOr -- guc: enable_seqscan = off\nSELECT id FROM orders WHERE memo @@> 'damaged' OR tenant_id = 5 S17 Search (BM25) BM25 index on a sift_table Yes BM25, sift_table SELECT count(*) FROM t WHERE tx @@> 'tx7' ambuild and scans run through the table access method. test:tbl_bm25 scan SELECT count(*) FROM cv_tt WHERE tx @@> 'tx7' S18 Search (BM25) Parallel scan of a match (Gather over SiftScan / SiftCount) Partly sift.parallel_scan = on (off by default); a ranked scan needs a LIMIT and one table; never with a pushed-down filter or over partitions BM25 SET sift.parallel_scan = on A Gather costs milliseconds: it paid only for heavy residual quals on 218,058 articles (1.8x). test:parallel_scan scan ~ Gather -- guc: sift.parallel_scan = on\n-- guc: max_parallel_workers_per_gather = 2\n-- guc: parallel_setup_cost = 0\n-- guc: parallel_tuple_cost = 0\n-- guc: min_parallel_table_scan_size = 0\n-- guc: min_parallel_index_scan_size = 0\nSELECT id FROM orders WHERE memo @@> 'damaged' AND length(memo) > 3 S19 Search (BM25) Hot standby, replication, VACUUM, REINDEX CONCURRENTLY, pg_dump Yes BM25 - MVCC, generic WAL, partial and expression indexes. Logical replication carries rows, not indexes: create the index on the subscriber. test:write_update,test:vacuum_converge true -- pre: VACUUM orders;\n-- pre: REINDEX INDEX CONCURRENTLY orders_fts;\nSELECT pg_get_indexdef('orders_fts'::regclass) LIKE '%USING sift (memo)%' AND coverage.plan_of('SELECT count(*) FROM orders WHERE memo @@> ''clean''') ~ 'Custom Scan \(Sift(Scan|Count)\)' S20 Search (BM25) Facets: sift.facet_counts(table, key, query, facets): terms with top N, range, histogram, date and array facets, several per call, multi-select Yes BM25 + sift_col SELECT * FROM sift.facet_counts('orders', 'memo', 'cable OR wire', '[{"expr":"source","size":5},{"expr":"total","edges":[100,300]}]') Exact counts of the same snapshot; each facet is its own scan, a Custom Scan (SiftAgg) when the columns are in the sift_col index, the normal plan (same counts) for an array facet or a column outside it. See FIELDS-FACETS.md. test:facet_counts ok SELECT count(*) FROM (SELECT value, count FROM sift.facet_counts('orders', 'memo', 'cable OR wire', '["source"]') WHERE kind = 'value' EXCEPT ALL SELECT source, count(*) FROM orders WHERE memo @@> 'cable OR wire' GROUP BY source) x C01 Search + analytics Aggregate over the rows of a match: GROUP BY with @@> in the WHERE Partly the table has a sift_col index with the grouped and aggregated columns; the planner takes the match set when its cost is lower, else SiftScan and a normal aggregate BM25 + sift_col SELECT source, count(*) FROM orders WHERE memo @@> 'cable' GROUP BY source The BM25 index gives the match set without reading the heap, the column index aggregates the matching rows (filters, generic aggregates, HAVING, parallel workers as for any SiftAgg); EXPLAIN shows Search: @@> . For counts per value of a keyword column, sift.facets also works. test:col_search,fixture:h_search_term,fixture:h_search_having agg ~ Search: orders_fts SELECT source, count(*) AS n, count(DISTINCT tenant_id) AS stores, min(placed_at) FROM orders WHERE memo @@> 'cable wire' AND deleted_at IS NULL GROUP BY source ORDER BY source C02 Search + analytics Aggregate over a ranked top-k (subquery with LIMIT) or a column aggregate joined to a ranked scan Yes BM25, sift_col SELECT status, count(*) FROM orders WHERE id IN (SELECT id FROM orders WHERE memo @@> 'damaged' ORDER BY sift.score(ctid) DESC LIMIT 100) GROUP BY 1 The ranked scan stays on SiftScan, as the plan under the node: its rows are read once when the aggregate starts and become a table of keys (a byte table for an integer or date key). The aggregate joined to the ranked scan with its score is the same join. The top-k must fit sift.agg_lookup_max_kb. fixture:pt_topk_in,fixture:pt_topk_join,fixture:pt_topk_scan,test:agg_plan_tables agg ~ Lookup Joins: (ANY_subquery|unnamed_subquery) \(subquery\) \(semi\) @pt_topk_in E01 Errors and limits sift.score() compared with a constant in WHERE (> >= < <=, either side): a minimum or maximum score Yes BM25 SELECT id FROM docs WHERE body @@> 'a' AND sift.score(ctid) > 1 The scan tests the cutoff on the score it computed, so the rows are those of a subquery that filters on the score; the score does not depend on the cutoff. A cutoff from below keeps the LIMIT hint of the ranked scan, one from above reads every match. A constant, a parameter or a stable expression of them; several cutoffs must all hold. BETWEEN and NOT (score > c) are such comparisons once PostgreSQL has rewritten them; OR, =, <>, an expression of the score, max_score() and a score of another query level stay 0A000: filter on the score from a subquery. test:rank_score_filter,test:rank_scan scan ~ Score Filter SELECT id FROM orders WHERE memo @@> 'damaged' AND sift.score(ctid) > 1 E02 Errors and limits sift.score() in UPDATE, DELETE, MERGE, FOR UPDATE, or a column with no sift index Error 0A000; rank in a SELECT and change the rows by key BM25 UPDATE t SET ... WHERE sift.score(ctid) > 0 test:rank_scan error:0A000 UPDATE orders SET status = status WHERE sift.score(ctid) > 0 AND memo @@> 'damaged' E03 Errors and limits NOT with no positive term: body @@> 'NOT wire' Error 42601; add a positive term BM25 SELECT id FROM docs WHERE body @@> 'NOT noisy' Also: queries over 256 leaves, nesting depth 32, phrases over 128 tokens, boosts over 10000. test:query_errors error:42601 SELECT id FROM orders WHERE memo @@> 'NOT wire' E04 Errors and limits BM25 index on several columns, a unique index, INCLUDE columns, a non-text key Error 0A000 (42704 for a non-text key); concatenate columns into one expression, cast the key to text BM25 CREATE INDEX ON docs USING sift (title, body) One key per index; no plain, index-only or ordered scans. test:edge_ddl error:0A000 CREATE INDEX cv_x ON orders USING sift (memo, status) E05 Errors and limits sift_col: unique, INCLUDE Error 0A000 sift_col CREATE UNIQUE INDEX ON t USING sift_col (id) test:col_index_basic error:0A000 CREATE UNIQUE INDEX cv_x ON cv_types USING sift_col (i4) L01 sift_table Aggregates over a sift_table: count, sum, avg, min, max, distinct, filters, grouping Yes sift_table SELECT grp, count(*), sum(i4) FROM t WHERE i4 > 100 GROUP BY grp The same SiftAgg node reads the column groups, the write buffer (as in-memory batches) and the tombstones with the table's own visibility; no index is needed. test:tbl_agg agg SELECT grp, count(*), sum(i4), min(d) FROM cv_tt WHERE i4 > 100 GROUP BY grp L02 sift_table Backward scan: a SCROLL cursor fetching backward, FETCH PRIOR / LAST Yes sift_table DECLARE c SCROLL CURSOR FOR SELECT * FROM t; FETCH LAST FROM c; FETCH PRIOR FROM c Both directions, as a heap: the first backward step lists the units of the range and walks them from the last; no unit is decoded twice per step. test:tbl_scroll true SELECT coverage.scroll_back('cv_tt') L03 sift_table More than 127 columns Error 54000 at CREATE TABLE and ALTER TABLE; at most 127 columns sift_table CREATE TABLE t (c1 int, ..., c128 int) USING sift_table ADD COLUMN past 127 fails as CREATE TABLE does; DROP COLUMN is not checked. test:tbl_limits error:54000 DO $d$ BEGIN EXECUTE 'CREATE TEMP TABLE cv_w128 (' || (SELECT string_agg('c' || g || ' int', ',') FROM generate_series(1, 128) g) || ') USING sift_table'; END $d$ L04 sift_table TID range scans (ctid >= ... AND ctid < ...) Yes sift_table SELECT count(*) FROM t WHERE ctid >= '(0,1)' AND ctid < '(100,1)' A plain row scan: the aggregate scan does not take a ctid qual. test:tbl_basic ok SELECT count(*) FROM cv_tt WHERE ctid >= '(0,1)' AND ctid < '(100,1)' L05 sift_table TABLESAMPLE SYSTEM and BERNOULLI Yes sift_table SELECT count(*) FROM t TABLESAMPLE BERNOULLI (10) REPEATABLE (7) A heap table with a column index keeps the normal plan for TABLESAMPLE (No in the aggregate rows). test:tbl_sample ok SELECT count(*) > 0 FROM cv_tt TABLESAMPLE BERNOULLI (50) L06 sift_table COPY in and out, INSERT ... SELECT, CREATE TABLE AS, ALTER TABLE ... SET ACCESS METHOD Yes sift_table COPY t FROM STDIN Rows go through the write buffer one at a time (no bulk column path). test:tbl_diff,test:tbl_basic true -- pre: CREATE TABLE cv_l06a (id int, v int) USING sift_table;\n-- pre: COPY cv_l06a FROM PROGRAM 'seq 1 1500 | sed -E "s/(.*)/\1,\1/"' (FORMAT csv);\n-- pre: COPY cv_l06a TO PROGRAM 'wc -l >/dev/null' (FORMAT csv);\n-- pre: CREATE TABLE cv_l06b (id int, v int) USING sift_table;\n-- pre: INSERT INTO cv_l06b SELECT i, i * 2 FROM generate_series(1, 1500) i;\n-- pre: CREATE TABLE cv_l06c USING sift_table AS SELECT i AS id, i * 3 AS v FROM generate_series(1, 1500) i;\n-- pre: CREATE TABLE cv_l06d AS SELECT i AS id, i * 4 AS v FROM generate_series(1, 1500) i;\n-- pre: ALTER TABLE cv_l06d SET ACCESS METHOD sift_table;\n-- pre: VACUUM cv_l06a, cv_l06b, cv_l06c, cv_l06d;\nSELECT bool_and(coverage.same(format('SELECT id %% 7, count(*), sum(v) FROM %s GROUP BY 1', t))) FROM unnest(ARRAY['cv_l06a', 'cv_l06b', 'cv_l06c', 'cv_l06d']) t L07 sift_table Row scans of some columns (SELECT a FROM t WHERE ...), btree, GIN, BRIN, BM25 indexes, bitmap scans Yes sift_table SELECT i4 FROM t WHERE i4 < 10 Works as for a heap; a table access method is not told which columns a query needs, so every row scan decodes whole units. Only the aggregate scan reads just its columns. test:tbl_basic,test:tbl_bm25 ok SELECT i4 FROM cv_tt WHERE i4 < 10 L08 sift_table UPDATE, DELETE, ON CONFLICT, row locks, triggers, savepoints, VACUUM FULL, CLUSTER Partly no HOT; shared row locks degrade to exclusive sift_table UPDATE t SET ... WHERE ... An UPDATE writes a new version and a tombstone, both visible to the aggregate scan under MVCC. test:tbl_basic,test:tbl_diff true -- pre: CREATE TABLE cv_l08 (id int PRIMARY KEY, g int, v int) USING sift_table;\n-- pre: INSERT INTO cv_l08 SELECT i, i % 7, i FROM generate_series(1, 2000) i;\n-- pre: UPDATE cv_l08 SET v = v + 1 WHERE id % 5 = 0;\n-- pre: DELETE FROM cv_l08 WHERE id % 11 = 0;\n-- pre: INSERT INTO cv_l08 SELECT i, 1, i FROM generate_series(1, 20) i ON CONFLICT (id) DO UPDATE SET v = excluded.v + 100;\n-- pre: BEGIN;\n-- pre: SELECT id FROM cv_l08 WHERE id < 5 FOR SHARE;\n-- pre: SAVEPOINT s1;\n-- pre: UPDATE cv_l08 SET v = -1 WHERE id < 3;\n-- pre: ROLLBACK TO s1;\n-- pre: SELECT id FROM cv_l08 WHERE id BETWEEN 10 AND 12 FOR UPDATE;\n-- pre: COMMIT;\n-- pre: CREATE FUNCTION cv_l08_trg() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN NEW.v := NEW.v + 1; RETURN NEW; END $$;\n-- pre: CREATE TRIGGER cv_l08_t BEFORE UPDATE ON cv_l08 FOR EACH ROW EXECUTE FUNCTION cv_l08_trg();\n-- pre: UPDATE cv_l08 SET v = v WHERE id % 13 = 0;\n-- pre: CREATE INDEX cv_l08_v ON cv_l08 (v);\n-- pre: VACUUM FULL cv_l08;\n-- pre: CLUSTER cv_l08 USING cv_l08_v;\nSELECT coverage.same('SELECT g, count(*), sum(v), min(v) FROM cv_l08 GROUP BY g') L09 sift_table System columns xmin, xmax, cmin, cmax Yes sift_table SELECT xmin FROM t ctid, tableoid, xmin, xmax, cmin and cmax are the row's: xmin and cmin the stored versions of its insertion (FrozenTransactionId once frozen), xmax and cmax those of its last delete, update or lock (0 if none; a row locked by several transactions shows the last locker, not a MultiXactId). test:tbl_sysattr true -- pre: CREATE TABLE cv_l09 (id int, v int) USING sift_table;\n-- pre: BEGIN;\n-- pre: INSERT INTO cv_l09 SELECT i, i FROM generate_series(1, 3) i;\n-- pre: CREATE TABLE cv_l09x AS SELECT pg_current_xact_id()::xid AS x;\n-- pre: COMMIT;\nSELECT bool_and(xmin = (SELECT x FROM cv_l09x)) AND bool_and(cmin::text = '0') AND count(*) FILTER (WHERE xmax::text <> '0') = 0 FROM cv_l09 L10 sift_table Publishing a sift_table under wal_level = logical without pg_sift in shared_preload_libraries Error 0A000 at CREATE or ALTER PUBLICATION (the event trigger sift_publication_check); preload pg_sift on the publisher and its standbys, or publish a heap staging table sift_table CREATE PUBLICATION p FOR TABLE t A subscriber needs nothing. Unpublished tables never need the preload. test:errors_hints external:logical CREATE PUBLICATION cv_p FOR TABLE cv_pub J28 Joins and subqueries An inner join that returns a column of the table, or a semi join, on an integer (int8 too) or date leaf: the keys of the loaded table are tested against the leaf before tuples are made Yes sift_col SELECT s.kind, count(*) FROM orders o JOIN tenants s ON s.id = o.tenant_id WHERE s.country_code = 'DE' GROUP BY s.kind The keys of the rows the table holds (its own conditions applied while loading) are a byte table over their range (a hash table when they span more than 4 million values), a conjunct of the scan: units the range misses are not read, the other rows are tested as a set before the combo cache is asked (sift.agg_join_filter, Join Key Filters of EXPLAIN ANALYZE). LEFT, RIGHT, FULL and anti joins and text keys keep the generic join, with the same rows. fixture:j_join_filter_inner,fixture:j_join_filter_two_joins,test:agg_join_filter,test:agg_join_key_types agg ~ Lookup Joins: tenants @j_join_filter_inner J29 Joins and subqueries UNION ALL arms (or an inlined CTE read several times) that join the same table with the same columns and conditions Yes sift_col SELECT 'a', s.kind, count(*) FROM orders o JOIN tenants s ON s.id = o.tenant_id WHERE s.country_code = 'US' GROUP BY s.kind UNION ALL SELECT 'b', s.kind, count(*) FROM orders o JOIN tenants s ON s.id = o.tenant_id WHERE s.country_code = 'US' AND o.deleted_at IS NULL GROUP BY s.kind The first arm loads the table into a context of the statement and the other arms use its rows (sift.agg_share_lookups, Shared Lookup Tables of EXPLAIN ANALYZE); arms that read other columns or conditions of the table, plans as tables and preserved sides load their own. A backend keeps the rows of a vacuumed, unchanged table for its next statement (sift.agg_lookup_cache_kb, Cached Lookup Tables). fixture:j_share_arms,test:agg_share_lookups,test:agg_lookup_cache ok @j_share_arms J30 Joins and subqueries An EXISTS over the elements of an array leaf, written with a sub-select that PostgreSQL pulls up Yes sift_col SELECT w.dim_a_id % 5, count(*) FROM t_work w WHERE EXISTS (SELECT * FROM LATERAL (SELECT sign, score FROM jsonb_to_recordset(doc_norm(w.doc) -> 'parts') AS o(category text, sign text, score float8) WHERE o.score IS NOT NULL) o WHERE o.sign = 'positive') GROUP BY 1 The sub-select is pulled up by PostgreSQL into the EXISTS: the sub-plan has the function, its conditions and the dead entry of the sub-select; the scan tests the elements as for the plain form. fixture:nf_exists_pulled_up,test:agg_nullable_filters ok @nf_exists_pulled_up J31 Joins and subqueries Conditions of the nullable input of an outer join that test the arrays of its rows (an EXISTS, a jsonpath test), and the aggregate filters that read a test computed inside it Yes a computed test that is true for an absent input (`x IS NULL`) keeps the normal plan sift_col SELECT s.kind, count(DISTINCT w.id) FILTER (WHERE w.managed IS TRUE) FROM tenants s LEFT JOIN (SELECT x.id, x.dim_a_id, jsonb_path_exists(x.comments, '$[*] ? (@.type == "type_a")') AS managed FROM t_work x WHERE x.deleted_at IS NULL) w ON w.dim_a_id = s.id WHERE s.id <= 40 GROUP BY 1 The ON conjuncts that read only the scanned table filter its rows before matching, like the conditions of a sub-select pulled up, and a placeholder computed inside the input is read through its guard (`CASE WHEN absent THEN NULL ELSE x END`) when the test is not true for an absent input. fixture:nf_left_derived_tests,fixture:nf_left_derived_absent_true,test:agg_nullable_filters ok @nf_left_derived_tests J32 Joins and subqueries A WITH inside a derived table (a ranked top-N over an aggregate, UNION ALL arms that read one CTE of it) Yes the derived table reads nothing of the queries around it (a lateral one that does keeps the normal plan) sift_col SELECT r.kind, r.n FROM (SELECT t.*, row_number() OVER (ORDER BY t.n DESC, t.kind) AS rn FROM (WITH scope AS MATERIALIZED (SELECT DISTINCT id FROM tenants WHERE country_code <> 'ZZ') SELECT s.kind, count(*) AS n FROM orders o JOIN scope c ON c.id = o.tenant_id JOIN tenants s ON s.id = c.id WHERE o.deleted_at IS NULL GROUP BY s.kind) t) r WHERE r.rn <= 3 The CTEs of such a derived table are lookup plans of the scan like the statement's own, and a CTE of it that several aggregates read is planned inline when the arms run on the index (the plan that costs less wins). fixture:nf_nested_cte_lookups,fixture:nf_nested_cte_shared,fixture:nf_nested_cte_correlated,test:agg_nested_ctes ok @nf_nested_cte_lookups J33 Joins and subqueries A join to a sub-select whose rows the plan of the sub-select already filters (the run condition of a ranking window, a top-N per group) Yes sift_col SELECT s.kind, count(*) FROM orders o JOIN tenants s ON s.id = o.tenant_id WHERE o.tenant_id IN (SELECT tenant_id FROM (SELECT tenant_id, row_number() OVER (PARTITION BY kind ORDER BY n DESC, tenant_id) AS rn FROM (SELECT o2.tenant_id, s2.kind, count(*) AS n FROM orders o2 JOIN tenants s2 ON s2.id = o2.tenant_id GROUP BY 1, 2) c) ranked WHERE rn <= 3) GROUP BY 1 A condition over the columns of a joined sub-select that PostgreSQL pushed down into it is enforced by its plan, which then does not output the column: the scan does not test it again (a column the query reads stays in the output, and the condition is tested as usual). fixture:nf_pushed_rank_semi,fixture:nf_pushed_rank_left,fixture:nf_pushed_rank_read,test:agg_pushed_conditions ok @nf_pushed_rank_semi J34 Joins and subqueries EXISTS over the elements of an array whose test reads a lookup per element (a mapping table by the element and a column of a joined table) or a column of a joined table Yes the sub-select reads the array of the row, a lookup LIMIT 1 on columns of the element and the joined table, and nothing that can raise; EXISTS in the WHERE (also under an outer join); no workers sift_col @ee_mapped_category Each element is looked up once per distinct value of the columns the lookup reads (PostgreSQL's plan of the lookup, run on the first need); a row passes when some element does, as in the normal plan, and the result for a row is kept while its elements and the column it reads are the same. NOT EXISTS, EXISTS inside an aggregate FILTER and expressions that can raise keep the normal plan. fixture:ee_mapped_category,fixture:ee_dimension_column,fixture:ee_unnested_rows,fixture:ee_calendar,test:agg_exists_elements agg @ee_mapped_category J35 Joins and subqueries IN, EXISTS, NOT EXISTS over a tree of joined tables (an inner join; a LEFT JOIN whose result gives the keys, behind a table of its own) Yes the keys come from one table of the tree; PostgreSQL planned the tree as a join of its own, uncorrelated, serial and without a volatile function, and its keys fit sift.agg_lookup_max_kb sift_col, sift_table SELECT status, count(*) FROM orders WHERE tenant_id IN (SELECT s.id FROM tenants s JOIN dim_country c ON c.country_code = s.country_code WHERE c.continent <> 'north') GROUP BY status The plan PostgreSQL made of the tree runs once when the scan starts and gives the key set (duplicates and NULL keys as in the normal plan, an empty set matches no row, an anti join keeps a NULL key); a condition of the ON clause on the tree's own columns is already in that plan. A key of an expression over two tables of the tree (`IN (SELECT coalesce(a.k, b.k) FROM a LEFT JOIN b ..)` straight under the scan) is not pushed down: behind a table of its own (`t.id IN (SELECT coalesce(..) ..)`) it is. fixture:j_semi_tree,fixture:j_semi_tree_left,test:agg_semi_tree agg @j_semi_tree J36 Joins and subqueries LEFT JOIN LATERAL over jsonb_to_recordset or jsonb_array_elements of a list of the row, ON true or with conditions of its own Yes a list of the scanned row (one level); a WHERE above the join that PostgreSQL could not make an inner join (it sees the NULL tuple), a lateral condition that may raise, or sift.agg_left_lateral off keeps the normal plan sift_col, sift_table SELECT o.id, max(o.total), avg(i.price) FROM orders o LEFT JOIN LATERAL jsonb_to_recordset(o.items) AS i(category text, price float8) ON true GROUP BY o.id A row none of whose elements passes (no list, an empty one, all failing the lateral's own conditions) is one tuple with NULL element columns, so a per-row group keeps its row; a row the snapshot does not see is no tuple. fixture:j_left_lateral,fixture:j_left_lateral_cond,fixture:j_left_lateral_rank,test:agg_left_lateral agg @j_left_lateral J37 Joins and subqueries FROM x, LATERAL (SELECT aggregates FROM f WHERE f.k = x.k) and JOIN LATERAL (..) ON true: the table grouped once by the tie column, joined to x Yes kept only when the planner costs it below the lookups of the normal plan; a sub-select that reads the columns of another lateral sub-select keeps the normal plan sift_col SELECT o.id, s.t FROM owners o, LATERAL (SELECT sum(i.amt) t FROM items i WHERE i.set_id = o.k) s An outer row whose key has no row gets NULL (0 for a count), a NULL key has none, a repeated outer row gets its group each time: the rows of the normal plan. 6000 outer rows over 200,000: 141-176 ms of lookups against 5.4-5.8 ms. test:agg_decorrelate agg -- pre: DROP TABLE IF EXISTS cov_items;\n-- pre: DROP TABLE IF EXISTS cov_owners;\n-- pre: CREATE TABLE cov_owners (id int PRIMARY KEY, k int) WITH (autovacuum_enabled = off);\n-- pre: INSERT INTO cov_owners SELECT i, CASE WHEN i % 13 = 0 THEN NULL ELSE i % 1500 END FROM generate_series(1, 2000) i;\n-- pre: CREATE TABLE cov_items (id int PRIMARY KEY, set_id int, cat int, amt int) WITH (autovacuum_enabled = off);\n-- pre: INSERT INTO cov_items SELECT i, CASE WHEN i % 17 = 0 THEN NULL ELSE i % 1200 END, i % 9, CASE WHEN i % 23 = 0 THEN NULL ELSE (i * 7) % 100 END FROM generate_series(1, 30000) i;\n-- pre: CREATE INDEX cov_items_col ON cov_items USING sift_col (id, set_id, cat, amt);\n-- pre: CREATE INDEX cov_items_owner ON cov_items (set_id);\n-- pre: VACUUM (FREEZE, ANALYZE) cov_owners;\n-- pre: VACUUM (FREEZE, ANALYZE) cov_items;\nSELECT o.id, s.t, s.c FROM cov_owners o, LATERAL (SELECT sum(i.amt) t, count(*) c FROM cov_items i WHERE i.set_id = o.k) s J38 Joins and subqueries LEFT JOIN LATERAL (SELECT g, aggregates FROM f WHERE f.k = x.k GROUP BY g [HAVING ..]) ON true (also JOIN and the comma form) Yes kept only when the planner costs the grouped table below the lookups of the normal plan sift_col SELECT o.id, s.cat, s.t FROM owners o LEFT JOIN LATERAL (SELECT i.cat, sum(i.amt) t FROM items i WHERE i.set_id = o.k GROUP BY i.cat) s ON true f is grouped by k and g and joined to x by k; an outer row with no group is one row with NULLs under LEFT JOIN and no row under JOIN, as in the normal plan. test:agg_decorrelate agg -- pre: DROP TABLE IF EXISTS cov_items;\n-- pre: DROP TABLE IF EXISTS cov_owners;\n-- pre: CREATE TABLE cov_owners (id int PRIMARY KEY, k int) WITH (autovacuum_enabled = off);\n-- pre: INSERT INTO cov_owners SELECT i, CASE WHEN i % 13 = 0 THEN NULL ELSE i % 1500 END FROM generate_series(1, 2000) i;\n-- pre: CREATE TABLE cov_items (id int PRIMARY KEY, set_id int, cat int, amt int) WITH (autovacuum_enabled = off);\n-- pre: INSERT INTO cov_items SELECT i, CASE WHEN i % 17 = 0 THEN NULL ELSE i % 1200 END, i % 9, CASE WHEN i % 23 = 0 THEN NULL ELSE (i * 7) % 100 END FROM generate_series(1, 30000) i;\n-- pre: CREATE INDEX cov_items_col ON cov_items USING sift_col (id, set_id, cat, amt);\n-- pre: CREATE INDEX cov_items_owner ON cov_items (set_id);\n-- pre: VACUUM (FREEZE, ANALYZE) cov_owners;\n-- pre: VACUUM (FREEZE, ANALYZE) cov_items;\nSELECT o.id, s.cat, s.t FROM cov_owners o LEFT JOIN LATERAL (SELECT i.cat, sum(i.amt) t FROM cov_items i WHERE i.set_id = o.k GROUP BY i.cat) s ON true