Skip to content

Commit 233a755

Browse files
gongxun0928jiaqizho
gongxun0928
authored and
jiaqizho
committed
PAX: Support zorder curve
Support multiple files to be rewritten in order according to the z-order curve Because the cluster implementation of Postgres depends on the index, we support two implementations in PAX AM: index cluster and column-based zorder cluster. Only one of them can be executed at the same time. The default sorting of zorder cluster is in ascending order of z-value. ``` -- zorder cluster create table t1(c1 int, c2 int) using pax with(cluster_columns='c1'); insert into t1 select i,i from generate_series(10,1,-1) i; table t1; cluster t1; table t1; ```
1 parent 9ce9c35 commit 233a755

38 files changed

Lines changed: 1751 additions & 387 deletions

‎contrib/pax_storage/expected/setup.out‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
CREATE OR REPLACE FUNCTION "get_pax_aux_table"(table_name text)
2+
RETURNS TABLE("ptblockname" name,"pttupcount" integer,
3+
"ptstatistics" pg_ext_aux.paxauxstats,
4+
"ptexistvisimap" bool, "ptexistexttoast" bool, "ptisclustered" bool) AS $BODY$
5+
DECLARE
6+
subquery varchar;
7+
pre_sql varchar;
8+
table_oid Oid;
9+
begin
10+
pre_sql:='select oid from pg_class where relname='''||table_name||'''';
11+
EXECUTE pre_sql into table_oid;
12+
subquery := 'select ptblockname, pttupcount, ptstatistics, ptvisimapname IS NOT NULL AS ptexistvisimap, ptexistexttoast, ptisclustered from gp_dist_random(''pg_ext_aux.pg_pax_blocks_'||table_oid||''')';
13+
RETURN QUERY execute subquery;
14+
END
15+
$BODY$
16+
LANGUAGE plpgsql;

‎contrib/pax_storage/expected/statistics.out‎

Lines changed: 288 additions & 304 deletions
Large diffs are not rendered by default.

‎contrib/pax_storage/expected/teardown.out‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
drop FUNCTION get_pax_aux_table;

‎contrib/pax_storage/expected/zorder_cluster.out‎

Lines changed: 169 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,169 @@
1+
-- test cluster_columns
2+
set pax_max_tuples_per_file to 131072;
3+
drop table if EXISTS t_zorder_cluster;
4+
create table t_zorder_cluster(c1 int, c2 int) using pax with(minmax_columns='c1,c2');
5+
\d+ t_zorder_cluster;
6+
Table "public.t_zorder_cluster"
7+
Column | Type | Collation | Nullable | Default | Storage | Stats target | Description
8+
--------+---------+-----------+----------+---------+---------+--------------+-------------
9+
c1 | integer | | | | plain | |
10+
c2 | integer | | | | plain | |
11+
Distributed by: (c1)
12+
Options: minmax_columns=c1,c2
13+
14+
insert into t_zorder_cluster select i,i from generate_series(1,100000) i;
15+
insert into t_zorder_cluster select i,i from generate_series(1,100000) i;
16+
insert into t_zorder_cluster select i,i from generate_series(1,100000) i;
17+
insert into t_zorder_cluster select i,i from generate_series(1,100000) i;
18+
insert into t_zorder_cluster select i,i from generate_series(1,100000) i;
19+
select ptblockname,ptstatistics,ptisclustered from get_pax_aux_table('t_zorder_cluster');
20+
ptblockname | ptstatistics | ptisclustered
21+
-------------+-------------------------------------------------------------------------------------------------+---------------
22+
0 | [(false,false),(33211),(5,100000),(1664130330)],[(false,false),(33211),(5,100000),(1664130330)] | f
23+
1 | [(false,false),(33211),(5,100000),(1664130330)],[(false,false),(33211),(5,100000),(1664130330)] | f
24+
2 | [(false,false),(33211),(5,100000),(1664130330)],[(false,false),(33211),(5,100000),(1664130330)] | f
25+
3 | [(false,false),(33211),(5,100000),(1664130330)],[(false,false),(33211),(5,100000),(1664130330)] | f
26+
4 | [(false,false),(33211),(5,100000),(1664130330)],[(false,false),(33211),(5,100000),(1664130330)] | f
27+
0 | [(false,false),(33462),(2,99999),(1674800729)],[(false,false),(33462),(2,99999),(1674800729)] | f
28+
1 | [(false,false),(33462),(2,99999),(1674800729)],[(false,false),(33462),(2,99999),(1674800729)] | f
29+
2 | [(false,false),(33462),(2,99999),(1674800729)],[(false,false),(33462),(2,99999),(1674800729)] | f
30+
3 | [(false,false),(33462),(2,99999),(1674800729)],[(false,false),(33462),(2,99999),(1674800729)] | f
31+
4 | [(false,false),(33462),(2,99999),(1674800729)],[(false,false),(33462),(2,99999),(1674800729)] | f
32+
0 | [(false,false),(33327),(1,99996),(1661118941)],[(false,false),(33327),(1,99996),(1661118941)] | f
33+
1 | [(false,false),(33327),(1,99996),(1661118941)],[(false,false),(33327),(1,99996),(1661118941)] | f
34+
2 | [(false,false),(33327),(1,99996),(1661118941)],[(false,false),(33327),(1,99996),(1661118941)] | f
35+
3 | [(false,false),(33327),(1,99996),(1661118941)],[(false,false),(33327),(1,99996),(1661118941)] | f
36+
4 | [(false,false),(33327),(1,99996),(1661118941)],[(false,false),(33327),(1,99996),(1661118941)] | f
37+
(15 rows)
38+
39+
-- error, becuase no cluster index or cluster_columns is defined
40+
cluster t_zorder_cluster;
41+
ERROR: there is no previously clustered index or cluster_columns reloptions for table "t_zorder_cluster"
42+
alter table t_zorder_cluster set(cluster_columns='c1,c2');
43+
\d+ t_zorder_cluster;
44+
Table "public.t_zorder_cluster"
45+
Column | Type | Collation | Nullable | Default | Storage | Stats target | Description
46+
--------+---------+-----------+----------+---------+---------+--------------+-------------
47+
c1 | integer | | | | plain | |
48+
c2 | integer | | | | plain | |
49+
Distributed by: (c1)
50+
Options: minmax_columns=c1,c2, cluster_columns=c1,c2
51+
52+
-- success
53+
cluster t_zorder_cluster;
54+
select ptblockname,ptstatistics,ptisclustered from get_pax_aux_table('t_zorder_cluster');
55+
ptblockname | ptstatistics | ptisclustered
56+
-------------+---------------------------------------------------------------------------------------------------------+---------------
57+
5 | [(false,false),(131072),(1,78633),(5126644321)],[(false,false),(131072),(1,78633),(5126644321)] | t
58+
6 | [(false,false),(35563),(78633,99996),(3178950384)],[(false,false),(35563),(78633,99996),(3178950384)] | t
59+
5 | [(false,false),(131072),(2,78230),(5147801190)],[(false,false),(131072),(2,78230),(5147801190)] | t
60+
6 | [(false,false),(36238),(78230,99999),(3226202455)],[(false,false),(36238),(78230,99999),(3226202455)] | t
61+
5 | [(false,false),(131072),(5,79072),(5187913809)],[(false,false),(131072),(5,79072),(5187913809)] | t
62+
6 | [(false,false),(34983),(79072,100000),(3132737841)],[(false,false),(34983),(79072,100000),(3132737841)] | t
63+
(6 rows)
64+
65+
drop table t_zorder_cluster;
66+
-- test cluster index
67+
set pax_max_tuples_per_file to 131072;
68+
drop table if EXISTS t_index_cluster;
69+
NOTICE: table "t_index_cluster" does not exist, skipping
70+
create table t_index_cluster(c1 int, c2 int) using pax with(minmax_columns='c1,c2');
71+
\d+ t_index_cluster;
72+
Table "public.t_index_cluster"
73+
Column | Type | Collation | Nullable | Default | Storage | Stats target | Description
74+
--------+---------+-----------+----------+---------+---------+--------------+-------------
75+
c1 | integer | | | | plain | |
76+
c2 | integer | | | | plain | |
77+
Distributed by: (c1)
78+
Options: minmax_columns=c1,c2
79+
80+
insert into t_index_cluster select i,i from generate_series(1,100000) i;
81+
insert into t_index_cluster select i,i from generate_series(1,100000) i;
82+
insert into t_index_cluster select i,i from generate_series(1,100000) i;
83+
insert into t_index_cluster select i,i from generate_series(1,100000) i;
84+
insert into t_index_cluster select i,i from generate_series(1,100000) i;
85+
select ptblockname,ptstatistics,ptisclustered from get_pax_aux_table('t_index_cluster');
86+
ptblockname | ptstatistics | ptisclustered
87+
-------------+-------------------------------------------------------------------------------------------------+---------------
88+
0 | [(false,false),(33327),(1,99996),(1661118941)],[(false,false),(33327),(1,99996),(1661118941)] | f
89+
1 | [(false,false),(33327),(1,99996),(1661118941)],[(false,false),(33327),(1,99996),(1661118941)] | f
90+
2 | [(false,false),(33327),(1,99996),(1661118941)],[(false,false),(33327),(1,99996),(1661118941)] | f
91+
3 | [(false,false),(33327),(1,99996),(1661118941)],[(false,false),(33327),(1,99996),(1661118941)] | f
92+
4 | [(false,false),(33327),(1,99996),(1661118941)],[(false,false),(33327),(1,99996),(1661118941)] | f
93+
0 | [(false,false),(33211),(5,100000),(1664130330)],[(false,false),(33211),(5,100000),(1664130330)] | f
94+
1 | [(false,false),(33211),(5,100000),(1664130330)],[(false,false),(33211),(5,100000),(1664130330)] | f
95+
2 | [(false,false),(33211),(5,100000),(1664130330)],[(false,false),(33211),(5,100000),(1664130330)] | f
96+
3 | [(false,false),(33211),(5,100000),(1664130330)],[(false,false),(33211),(5,100000),(1664130330)] | f
97+
4 | [(false,false),(33211),(5,100000),(1664130330)],[(false,false),(33211),(5,100000),(1664130330)] | f
98+
0 | [(false,false),(33462),(2,99999),(1674800729)],[(false,false),(33462),(2,99999),(1674800729)] | f
99+
1 | [(false,false),(33462),(2,99999),(1674800729)],[(false,false),(33462),(2,99999),(1674800729)] | f
100+
2 | [(false,false),(33462),(2,99999),(1674800729)],[(false,false),(33462),(2,99999),(1674800729)] | f
101+
3 | [(false,false),(33462),(2,99999),(1674800729)],[(false,false),(33462),(2,99999),(1674800729)] | f
102+
4 | [(false,false),(33462),(2,99999),(1674800729)],[(false,false),(33462),(2,99999),(1674800729)] | f
103+
(15 rows)
104+
105+
-- error, becuase no cluster index or cluster_columns is defined
106+
cluster t_index_cluster;
107+
ERROR: there is no previously clustered index or cluster_columns reloptions for table "t_index_cluster"
108+
create index idx_t_index_cluster on t_index_cluster(c1);
109+
\d+ t_index_cluster;
110+
Table "public.t_index_cluster"
111+
Column | Type | Collation | Nullable | Default | Storage | Stats target | Description
112+
--------+---------+-----------+----------+---------+---------+--------------+-------------
113+
c1 | integer | | | | plain | |
114+
c2 | integer | | | | plain | |
115+
Indexes:
116+
"idx_t_index_cluster" btree (c1)
117+
Distributed by: (c1)
118+
Options: minmax_columns=c1,c2
119+
120+
-- success
121+
cluster t_index_cluster using idx_t_index_cluster;
122+
select ptblockname,ptstatistics,ptisclustered from get_pax_aux_table('t_index_cluster');
123+
ptblockname | ptstatistics | ptisclustered
124+
-------------+---------------------------------------------------------------------------------------------------------+---------------
125+
0 | [(false,false),(131072),(2,78230),(5147801190)],[(false,false),(131072),(2,78230),(5147801190)] | t
126+
1 | [(false,false),(36238),(78230,99999),(3226202455)],[(false,false),(36238),(78230,99999),(3226202455)] | t
127+
0 | [(false,false),(131072),(1,78633),(5126644321)],[(false,false),(131072),(1,78633),(5126644321)] | t
128+
1 | [(false,false),(35563),(78633,99996),(3178950384)],[(false,false),(35563),(78633,99996),(3178950384)] | t
129+
0 | [(false,false),(131072),(5,79072),(5187913809)],[(false,false),(131072),(5,79072),(5187913809)] | t
130+
1 | [(false,false),(34983),(79072,100000),(3132737841)],[(false,false),(34983),(79072,100000),(3132737841)] | t
131+
(6 rows)
132+
133+
drop table t_index_cluster;
134+
-- test both cluster index and cluster columns
135+
create table t_both_zorder_and_index_cluster(c1 int, c2 int) using pax with(minmax_columns='c1,c2');
136+
alter table t_both_zorder_and_index_cluster set(cluster_columns='c1,c2');
137+
create index idx_t_both_zorder_and_index_cluster on t_both_zorder_and_index_cluster(c1);
138+
-- error, because zorder cluster and index cluster can not be used together
139+
cluster t_both_zorder_and_index_cluster using idx_t_both_zorder_and_index_cluster;
140+
ERROR: cannot using index to cluster table which has cluster-columns (pax_access_handle.cc:1186)
141+
-- success
142+
cluster t_both_zorder_and_index_cluster;
143+
alter table t_both_zorder_and_index_cluster set(cluster_columns='');
144+
-- error, because zorder cluster reloptions has been deleted
145+
cluster t_both_zorder_and_index_cluster;
146+
ERROR: there is no previously clustered index or cluster_columns reloptions for table "t_both_zorder_and_index_cluster"
147+
-- success
148+
cluster t_both_zorder_and_index_cluster using idx_t_both_zorder_and_index_cluster;
149+
-- error
150+
alter table t_both_zorder_and_index_cluster set(cluster_columns='c1,c2');
151+
ERROR: pax table has previously clustered index, can't set zorder cluster reloptions (pax_access_handle.cc:1308)
152+
drop index idx_t_both_zorder_and_index_cluster;
153+
-- error
154+
cluster t_both_zorder_and_index_cluster;
155+
ERROR: there is no previously clustered index or cluster_columns reloptions for table "t_both_zorder_and_index_cluster"
156+
-- success
157+
alter table t_both_zorder_and_index_cluster set(cluster_columns='c1,c2');
158+
-- success
159+
cluster t_both_zorder_and_index_cluster;
160+
-- test unsupport type
161+
create table t_zorder_unsupport_type(c1 int, c2 numeric(10,2),c3 varchar(128), c4 timestamp, c5 bpchar(64) ) using pax with(minmax_columns='c1,c2');
162+
-- error, because numeric is unsupport type
163+
alter table t_zorder_unsupport_type set(cluster_columns='c1,c2');
164+
ERROR: the type of column c2 does not support zorder cluster (paxc_rel_options.cc:307)
165+
alter table t_zorder_unsupport_type set(cluster_columns='c1,c3');
166+
-- error, because timestamp is unsupport type
167+
alter table t_zorder_unsupport_type set(cluster_columns='c1,c4');
168+
ERROR: the type of column c4 does not support zorder cluster (paxc_rel_options.cc:307)
169+
alter table t_zorder_unsupport_type set(cluster_columns='c1,c5');

‎contrib/pax_storage/pax_schedule‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,4 +8,6 @@ test: statistics
88
test: visimap_vec_compact
99
test: visimap_vec_storage
1010

11+
test: zorder_cluster
12+
1113
test: teardown

‎contrib/pax_storage/sql/setup.sql‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,3 +2,20 @@
22
create schema pax_test;
33
CREATE EXTENSION gp_inject_fault;
44
-- end_ignore
5+
6+
CREATE OR REPLACE FUNCTION "get_pax_aux_table"(table_name text)
7+
RETURNS TABLE("ptblockname" name,"pttupcount" integer,
8+
"ptstatistics" pg_ext_aux.paxauxstats,
9+
"ptexistvisimap" bool, "ptexistexttoast" bool, "ptisclustered" bool) AS $BODY$
10+
DECLARE
11+
subquery varchar;
12+
pre_sql varchar;
13+
table_oid Oid;
14+
begin
15+
pre_sql:='select oid from pg_class where relname='''||table_name||'''';
16+
EXECUTE pre_sql into table_oid;
17+
subquery := 'select ptblockname, pttupcount, ptstatistics, ptvisimapname IS NOT NULL AS ptexistvisimap, ptexistexttoast, ptisclustered from gp_dist_random(''pg_ext_aux.pg_pax_blocks_'||table_oid||''')';
18+
RETURN QUERY execute subquery;
19+
END
20+
$BODY$
21+
LANGUAGE plpgsql;

‎contrib/pax_storage/sql/statistics.sql‎

Lines changed: 0 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,5 @@
11
set default_table_access_method = pax;
22

3-
CREATE OR REPLACE FUNCTION "get_pax_aux_table"(table_name text)
4-
RETURNS TABLE("ptblockname" name,"pttupcount" integer,
5-
"ptstatistics" pg_ext_aux.paxauxstats,
6-
"ptexistvisimap" bool, "ptexistexttoast" bool) AS $BODY$
7-
DECLARE
8-
subquery varchar;
9-
pre_sql varchar;
10-
table_oid Oid;
11-
begin
12-
pre_sql:='select oid from pg_class where relname='''||table_name||'''';
13-
EXECUTE pre_sql into table_oid;
14-
subquery := 'select ptblockname, pttupcount, ptstatistics, ptvisimapname IS NOT NULL AS ptexistvisimap, ptexistexttoast from gp_dist_random(''pg_ext_aux.pg_pax_blocks_'||table_oid||''')';
15-
RETURN QUERY execute subquery;
16-
END
17-
$BODY$
18-
LANGUAGE plpgsql;
193

204
--
215
-- Test with small group

‎contrib/pax_storage/sql/teardown.sql‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
drop FUNCTION get_pax_aux_table;
12
-- start_ignore
23
drop schema if exists pax_test;
34
-- end_ignore

‎contrib/pax_storage/sql/zorder_cluster.sql‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
-- test cluster_columns
2+
set pax_max_tuples_per_file to 131072;
3+
drop table if EXISTS t_zorder_cluster;
4+
create table t_zorder_cluster(c1 int, c2 int) using pax with(minmax_columns='c1,c2');
5+
\d+ t_zorder_cluster;
6+
7+
insert into t_zorder_cluster select i,i from generate_series(1,100000) i;
8+
insert into t_zorder_cluster select i,i from generate_series(1,100000) i;
9+
insert into t_zorder_cluster select i,i from generate_series(1,100000) i;
10+
insert into t_zorder_cluster select i,i from generate_series(1,100000) i;
11+
insert into t_zorder_cluster select i,i from generate_series(1,100000) i;
12+
13+
select ptblockname,ptstatistics,ptisclustered from get_pax_aux_table('t_zorder_cluster');
14+
15+
-- error, becuase no cluster index or cluster_columns is defined
16+
cluster t_zorder_cluster;
17+
18+
alter table t_zorder_cluster set(cluster_columns='c1,c2');
19+
\d+ t_zorder_cluster;
20+
-- success
21+
cluster t_zorder_cluster;
22+
select ptblockname,ptstatistics,ptisclustered from get_pax_aux_table('t_zorder_cluster');
23+
24+
25+
drop table t_zorder_cluster;
26+
27+
-- test cluster index
28+
set pax_max_tuples_per_file to 131072;
29+
drop table if EXISTS t_index_cluster;
30+
create table t_index_cluster(c1 int, c2 int) using pax with(minmax_columns='c1,c2');
31+
\d+ t_index_cluster;
32+
33+
insert into t_index_cluster select i,i from generate_series(1,100000) i;
34+
insert into t_index_cluster select i,i from generate_series(1,100000) i;
35+
insert into t_index_cluster select i,i from generate_series(1,100000) i;
36+
insert into t_index_cluster select i,i from generate_series(1,100000) i;
37+
insert into t_index_cluster select i,i from generate_series(1,100000) i;
38+
39+
select ptblockname,ptstatistics,ptisclustered from get_pax_aux_table('t_index_cluster');
40+
41+
-- error, becuase no cluster index or cluster_columns is defined
42+
cluster t_index_cluster;
43+
44+
create index idx_t_index_cluster on t_index_cluster(c1);
45+
\d+ t_index_cluster;
46+
47+
-- success
48+
cluster t_index_cluster using idx_t_index_cluster;
49+
50+
select ptblockname,ptstatistics,ptisclustered from get_pax_aux_table('t_index_cluster');
51+
drop table t_index_cluster;
52+
53+
-- test both cluster index and cluster columns
54+
create table t_both_zorder_and_index_cluster(c1 int, c2 int) using pax with(minmax_columns='c1,c2');
55+
alter table t_both_zorder_and_index_cluster set(cluster_columns='c1,c2');
56+
create index idx_t_both_zorder_and_index_cluster on t_both_zorder_and_index_cluster(c1);
57+
-- error, because zorder cluster and index cluster can not be used together
58+
cluster t_both_zorder_and_index_cluster using idx_t_both_zorder_and_index_cluster;
59+
-- success
60+
cluster t_both_zorder_and_index_cluster;
61+
alter table t_both_zorder_and_index_cluster set(cluster_columns='');
62+
-- error, because zorder cluster reloptions has been deleted
63+
cluster t_both_zorder_and_index_cluster;
64+
-- success
65+
cluster t_both_zorder_and_index_cluster using idx_t_both_zorder_and_index_cluster;
66+
-- error
67+
alter table t_both_zorder_and_index_cluster set(cluster_columns='c1,c2');
68+
drop index idx_t_both_zorder_and_index_cluster;
69+
-- error
70+
cluster t_both_zorder_and_index_cluster;
71+
-- success
72+
alter table t_both_zorder_and_index_cluster set(cluster_columns='c1,c2');
73+
-- success
74+
cluster t_both_zorder_and_index_cluster;
75+
76+
-- test unsupport type
77+
78+
create table t_zorder_unsupport_type(c1 int, c2 numeric(10,2),c3 varchar(128), c4 timestamp, c5 bpchar(64) ) using pax with(minmax_columns='c1,c2');
79+
80+
-- error, because numeric is unsupport type
81+
alter table t_zorder_unsupport_type set(cluster_columns='c1,c2');
82+
alter table t_zorder_unsupport_type set(cluster_columns='c1,c3');
83+
-- error, because timestamp is unsupport type
84+
alter table t_zorder_unsupport_type set(cluster_columns='c1,c4');
85+
alter table t_zorder_unsupport_type set(cluster_columns='c1,c5');

0 commit comments

Comments
 (0)