Skip to content

Commit ea5a33f

Browse files
Xiaoran Wangreshke
Xiaoran Wang
authored and
reshke
committed
Add upgrade job on pipeline(#60)
Add the upgrade job in the pipeline This job tests the extension upgrading from 1.0 to 2.0.
1 parent 174fc69 commit ea5a33f

38 files changed

Lines changed: 928 additions & 40 deletions

‎gpcontrib/diskquota/.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,3 +4,6 @@
44
regression.out
55
regression.diffs
66
/results/
7+
upgrade_test/regression.out
8+
upgrade_test/regression.diffs
9+
upgrade_test/results

‎gpcontrib/diskquota/concourse/pipeline/pipeline.yml‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,20 @@
33
## ======================================================================
44

55
groups:
6+
- name: ALL
7+
jobs:
8+
- diskquota_centos6_build_test
9+
- diskquota_centos7_build_test
10+
- diskquota_ubuntu18_build_test
11+
- diskquota_centos7_extension_upgrade_1.0_2.0
612
- name: GPDB6
713
jobs:
814
- diskquota_centos6_build_test
915
- diskquota_centos7_build_test
1016
- diskquota_ubuntu18_build_test
17+
- name: GPDB6_UPGRADE
18+
jobs:
19+
- diskquota_centos7_extension_upgrade_1.0_2.0
1120

1221
resource_types:
1322
- name: gcs
@@ -43,6 +52,20 @@ resources:
4352
repository: gcr.io/data-gpdb-public-images/gpdb6-ubuntu18.04-test
4453
tag: latest
4554

55+
- name: bin_diskquota_centos7_1.0
56+
type: gcs
57+
source:
58+
bucket: {{gcs-bucket}}
59+
json_key: {{concourse-gcs-resources-service-account-key}}
60+
regexp: diskquota/released/gpdb6/diskquota-(1.*)-rhel7_x86_64.tar.gz
61+
62+
- name: bin_diskquota_centos7
63+
type: gcs
64+
source:
65+
bucket: {{gcs-bucket-dev}}
66+
json_key: {{concourse-gcs-resources-service-account-key}}
67+
versioned_file: diskquota/intermediates/gpdb6/diskquota-master-rhel7_x86_64.tar.gz
68+
4669
# Github Source Codes
4770

4871
- name: gpdb_src
@@ -105,6 +128,10 @@ jobs:
105128
bin_diskquota: diskquota_artifacts
106129
params:
107130
DISKQUOTA_OS: rhel7
131+
- aggregate:
132+
- put: bin_diskquota_centos7
133+
params:
134+
file: diskquota_artifacts/diskquota*.tar.gz
108135

109136
- name: diskquota_centos6_build_test
110137
max_in_flight: 3
@@ -152,3 +179,26 @@ jobs:
152179
bin_diskquota: diskquota_artifacts
153180
params:
154181
DISKQUOTA_OS: ubuntu18.04
182+
183+
- name: diskquota_centos7_extension_upgrade_1.0_2.0
184+
max_in_flight: 3
185+
plan:
186+
- aggregate:
187+
- get: centos-gpdb-dev-7
188+
- get: bin_diskquota_old
189+
resource: bin_diskquota_centos7_1.0
190+
- get: bin_diskquota_centos7
191+
trigger: true
192+
- get: bin_gpdb
193+
resource: bin_gpdb_centos7
194+
- get: gpdb_src
195+
- get: diskquota_src
196+
- task: upgrade_extension
197+
file: diskquota_src/concourse/tasks/upgrade_extension.yml
198+
input_mapping:
199+
bin_diskquota_new: bin_diskquota_centos7
200+
image: centos-gpdb-dev-7
201+
params:
202+
DISKQUOTA_OS: rhel7
203+
OLD_VERSION: "1.0"
204+
NEW_VERSION: "2.0"

‎gpcontrib/diskquota/concourse/scripts/test_common.sh‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
# the directory to run the "make install" as the param
2+
function test(){
3+
chown -R gpadmin:gpadmin ${TOP_DIR};
4+
cat > /home/gpadmin/test.sh <<-EOF
5+
set -exo pipefail
6+
source gpdb_src/gpAux/gpdemo/gpdemo-env.sh
7+
echo "export MASTER_DATA_DIRECTORY=\$MASTER_DATA_DIRECTORY" >> /usr/local/greenplum-db-devel/greenplum_path.sh
8+
source /usr/local/greenplum-db-devel/greenplum_path.sh
9+
createdb diskquota
10+
gpconfig -c shared_preload_libraries -v 'diskquota'
11+
gpstop -arf
12+
gpconfig -c diskquota.naptime -v 1
13+
gpstop -arf
14+
# the dir to run the "make install" command
15+
pushd $1
16+
trap "[ -s regression.diffs ] && grep -v GP_IGNORE regression.diffs" EXIT
17+
make installcheck
18+
[ -s regression.diffs ] && grep -v GP_IGNORE regression.diffs && exit 1
19+
ps -ef | grep postgres| grep qddir| cut -d ' ' -f ${CUT_NUMBER} | xargs kill -9
20+
export PGPORT=6001
21+
echo "export PGPROT=\$PGPORT" >> /usr/local/greenplum-db-devel/greenplum_path.sh
22+
source /usr/local/greenplum-db-devel/greenplum_path.sh
23+
rm /tmp/.s.PGSQL.6000*
24+
gpactivatestandby -ad ${TOP_DIR}/gpdb_src/gpAux/gpdemo/datadirs/standby
25+
make installcheck
26+
[ -s regression.diffs ] && grep -v GP_IGNORE regression.diffs && exit 1
27+
popd
28+
EOF
29+
export MASTER_DATA_DIRECTORY=${TOP_DIR}/gpdb_src/gpAux/gpdemo/datadirs/qddir/demoDataDir-1
30+
chown gpadmin:gpadmin /home/gpadmin/test.sh
31+
chmod a+x /home/gpadmin/test.sh
32+
su gpadmin -c "bash /home/gpadmin/test.sh"
33+
}
34+
35+
function setup_gpadmin_user() {
36+
${GPDB_CONCOURSE_DIR}/setup_gpadmin_user.bash
37+
}
38+
39+
function install_diskquota() {
40+
tar -xzf bin_diskquota/*.tar.gz -C /usr/local/greenplum-db-devel
41+
}

‎gpcontrib/diskquota/concourse/scripts/test_diskquota.sh‎

Lines changed: 2 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -8,45 +8,7 @@ GPDB_CONCOURSE_DIR=${TOP_DIR}/gpdb_src/concourse/scripts
88
CUT_NUMBER=5
99

1010
source "${GPDB_CONCOURSE_DIR}/common.bash"
11-
function test(){
12-
chown -R gpadmin:gpadmin ${TOP_DIR};
13-
cat > /home/gpadmin/test.sh <<-EOF
14-
set -exo pipefail
15-
source gpdb_src/gpAux/gpdemo/gpdemo-env.sh
16-
echo "export MASTER_DATA_DIRECTORY=\$MASTER_DATA_DIRECTORY" >> /usr/local/greenplum-db-devel/greenplum_path.sh
17-
source /usr/local/greenplum-db-devel/greenplum_path.sh
18-
createdb diskquota
19-
gpconfig -c shared_preload_libraries -v 'diskquota'
20-
gpstop -arf
21-
gpconfig -c diskquota.naptime -v 1
22-
gpstop -arf
23-
pushd diskquota_src
24-
trap "[ -s regression.diffs ] && grep -v GP_IGNORE regression.diffs" EXIT
25-
make installcheck
26-
[ -s regression.diffs ] && grep -v GP_IGNORE regression.diffs && exit 1
27-
ps -ef | grep postgres| grep qddir| cut -d ' ' -f ${CUT_NUMBER} | xargs kill -9
28-
export PGPORT=6001
29-
echo "export PGPROT=\$PGPORT" >> /usr/local/greenplum-db-devel/greenplum_path.sh
30-
source /usr/local/greenplum-db-devel/greenplum_path.sh
31-
rm /tmp/.s.PGSQL.6000*
32-
gpactivatestandby -ad ${TOP_DIR}/gpdb_src/gpAux/gpdemo/datadirs/standby
33-
make installcheck
34-
[ -s regression.diffs ] && grep -v GP_IGNORE regression.diffs && exit 1
35-
popd
36-
EOF
37-
export MASTER_DATA_DIRECTORY=${TOP_DIR}/gpdb_src/gpAux/gpdemo/datadirs/qddir/demoDataDir-1
38-
chown gpadmin:gpadmin /home/gpadmin/test.sh
39-
chmod a+x /home/gpadmin/test.sh
40-
su gpadmin -c "bash /home/gpadmin/test.sh"
41-
}
42-
43-
function setup_gpadmin_user() {
44-
${GPDB_CONCOURSE_DIR}/setup_gpadmin_user.bash
45-
}
46-
47-
function install_diskquota() {
48-
tar -xzf bin_diskquota/*.tar.gz -C /usr/local/greenplum-db-devel
49-
}
11+
source "${TOP_DIR}/diskquota_src/concourse/scripts/test_common.sh"
5012
function _main() {
5113
time install_gpdb
5214
time setup_gpadmin_user
@@ -57,7 +19,7 @@ function _main() {
5719
CUT_NUMBER=6
5820
fi
5921

60-
time test
22+
time test ${TOP_DIR}/diskquota_src/
6123
}
6224

6325
_main "$@"

‎gpcontrib/diskquota/concourse/scripts/upgrade_extension.sh‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
#!/bin/bash -l
2+
3+
set -exo pipefail
4+
5+
CWDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
6+
TOP_DIR=${CWDIR}/../../../
7+
GPDB_CONCOURSE_DIR=${TOP_DIR}/gpdb_src/concourse/scripts
8+
CUT_NUMBER=5
9+
10+
source "${GPDB_CONCOURSE_DIR}/common.bash"
11+
source "${TOP_DIR}/diskquota_src/concourse/scripts/test_common.sh"
12+
13+
function install_old_version_diskquota() {
14+
tar -xzf bin_diskquota_old/*.tar.gz -C /usr/local/greenplum-db-devel
15+
}
16+
17+
# this function is called by upgrade_test/sql/upgrade_extension.sql
18+
function install_new_version_diskquota() {
19+
# the current dir is upgrade_test
20+
tar -xzf ../../bin_diskquota_new/*.tar.gz -C /usr/local/greenplum-db-devel
21+
}
22+
23+
function _main() {
24+
time install_gpdb
25+
time setup_gpadmin_user
26+
27+
time make_cluster
28+
if [ "${DISKQUOTA_OS}" == "ubuntu18.04" ]; then
29+
CUT_NUMBER=6
30+
fi
31+
32+
# install old_version diskquota
33+
time install_old_version_diskquota
34+
# export install_new_version_diskquota function, becuase it will
35+
# be called by upgrade_test/sql/upgrade_extension.sql
36+
export -f install_new_version_diskquota
37+
time test ${TOP_DIR}/diskquota_src/upgrade_test
38+
}
39+
40+
_main "$@"

‎gpcontrib/diskquota/concourse/tasks/upgrade_extension.yml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
platform: linux
2+
image_resource:
3+
type: docker-image
4+
inputs:
5+
- name: bin_gpdb
6+
- name: bin_diskquota_old
7+
- name: bin_diskquota_new
8+
- name: gpdb_src
9+
- name: diskquota_src
10+
11+
run:
12+
path: diskquota_src/concourse/scripts/upgrade_extension.sh
13+
params:
14+
DISKQUOTA_OS:
15+
OLD_VERSION:
16+
NEW_VERSION:

‎gpcontrib/diskquota/upgrade_test/Makefile‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
REGRESS = dummy
2+
3+
REGRESS_OPTS = --schedule=diskquota_schedule_upgrade --init-file=init_file
4+
PGXS := $(shell pg_config --pgxs)
5+
include $(PGXS)

‎gpcontrib/diskquota/upgrade_test/README.md‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
# upgrade_extension test
2+
The upgrade_extension test case will fail if
3+
run it locally. Because it calls
4+
"install_new_version_diskquota" function which is
5+
defined in concourse/scripts/upgrade_extension.sh.
6+
You can write this function by yourself and
7+
export it locally if you want to run it successfully.

‎gpcontrib/diskquota/upgrade_test/diskquota_schedule_upgrade‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
# run by old version diskquota
2+
test: init
3+
test: prepare
4+
test: set_config
5+
# upgrade diskquota to new version
6+
test: install_new_version
7+
test: upgrade_extension
8+
# run by new version diskquota
9+
test: test_role test_schema test_reschema test_temp_role test_rename test_delete_quota
10+
test: clean

‎gpcontrib/diskquota/upgrade_test/expected/clean.out‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
DROP TABLE badquota.t1;
2+
DROP ROLE testbody;
3+
DROP SCHEMA badquota;
4+
DROP EXTENSION diskquota;

0 commit comments

Comments
 (0)