## PRO TIP : Test your modifications locally with: ## $ gitlab-ci-multi-runner exec docker {name_of_the_job} variables: PGDATA: /var/lib/postgresql/data PGUSER: postgres EXTDIR: /usr/share/postgresql/13/extension/ PSQL: psql -v ON_ERROR_STOP=1 POSTGRES_DB: nice_marmot POSTGRES_USER: runner POSTGRES_PASSWORD: plop SAST_EXCLUDED_PATHS: '_venv' stages: - lint - build - test - deploy image: postgres:13 ## ## L I N T ## lint-bash: stage: lint script: - apt-get update - apt-get install -y --no-install-recommends shellcheck - shellcheck bin/standalone.sh - shellcheck bin/pg_dump_anon.sh - shellcheck docker/anon.sh lint-markdown: stage: lint image: ruby:alpine script: - gem install mdl - mdl docs/*.md *.md lint-py: stage: lint image: python:3.7 script: - pip3 install flake8 - flake8 python/*.py ## ## B U I L D ## # We need to launch manually the instance because the entrypoint is skipped. .init_PG_instance: &init_PG_instance before_script: - mkdir -p $PGDATA - mkdir -p $EXTDIR - chown postgres $PGDATA - gosu postgres initdb - gosu postgres pg_ctl start PG9.6-debian10: <<: *init_PG_instance stage: build image: postgres:9.6-buster script: - apt-get update - apt-get install -y --no-install-recommends build-essential postgresql-server-dev-9.6 pgxnclient python3-pip #- pip3 install --upgrade pip - pip3 install --no-cache-dir -r python/requirements.txt - make extension - make install - psql -c "ALTER SYSTEM SET session_preload_libraries = 'anon'" - psql -c "SELECT pg_reload_conf();" - make installcheck || diff results tests/expected artifacts: paths: - anon* - regression.* - results/ expire_in: 1 day PG10-debian10: <<: *init_PG_instance stage: build image: postgres:10-buster script: - apt-get update - apt-get install -y --no-install-recommends build-essential postgresql-server-dev-10 pgxnclient python3-pip - pip3 install --upgrade pip - pip3 install --no-cache-dir -r python/requirements.txt - make extension - make install - psql -c "ALTER SYSTEM SET session_preload_libraries = 'anon'" - psql -c "SELECT pg_reload_conf();" - make installcheck || diff results tests/expected when: manual artifacts: paths: - anon* - regression.* - results/ PG10-rocky8: stage: build image: rockylinux/rockylinux:8 script: - dnf -y install https://download.postgresql.org/pub/repos/yum/reporpms/EL-8-x86_64/pgdg-redhat-repo-latest.noarch.rpm - dnf -qy module disable postgresql - dnf -y install git make gcc redhat-rpm-config postgresql10 postgresql10-server postgresql10-contrib postgresql10-devel python3-pip diffutils - pip3 install --upgrade pip - pip3 install --no-cache-dir -r python/requirements.txt - export PATH="$PATH:/usr/pgsql-10/bin" - make - make install - mkdir -p $PGDATA - chown postgres $PGDATA - su postgres -c 'initdb' - su postgres -c 'pg_ctl start' - psql -c "ALTER SYSTEM SET session_preload_libraries = 'anon'" - psql -c "SELECT pg_reload_conf();" - make installcheck || diff results tests/expected artifacts: paths: - anon* - regression.* - results/ expire_in: 1 day PG11-debian10: <<: *init_PG_instance stage: build image: postgres:11-buster script: - apt-get update - apt-get install -y --no-install-recommends make gcc postgresql-server-dev-11 pgxnclient python3-pip - pip3 install --upgrade pip - pip3 install --no-cache-dir -r python/requirements.txt - make extension - make install - psql -c "ALTER SYSTEM SET session_preload_libraries = 'anon'" - psql -c "SELECT pg_reload_conf();" - make installcheck || diff results tests/expected artifacts: paths: - anon* - regression.* - results/ expire_in: 1 day ## DOES NOT WORK ? PG11-centos7: stage: build image: centos:7 script: # http://blog.cloud-mes.com/2020/05/16/fix-yum-update-postgresql12-to-v12-dot-3-require-llvm-toolset-7-clang-equals-4-dot-0-1-dependency-problem/ - yum -y install centos-release-scl-rh - yum -y install llvm-toolset-7-clang - yum -y install https://download.postgresql.org/pub/repos/yum/reporpms/EL-7-x86_64/pgdg-redhat-repo-latest.noarch.rpm - yum -y install git make gcc postgresql11 postgresql11-server postgresql11-devel postgresql11-contrib python3-pip diffutils - pip3 install --upgrade pip - pip3 install --no-cache-dir -r python/requirements.txt - export PATH="$PATH:/usr/pgsql-11/bin" - make - make install - mkdir -p $PGDATA - chown postgres $PGDATA - su postgres -c 'initdb' - su postgres -c 'pg_ctl start' - psql -c "ALTER SYSTEM SET session_preload_libraries = 'anon'" - psql -c "SELECT pg_reload_conf();" - make installcheck || diff results tests/expected when: manual artifacts: paths: - anon* - regression.* - results/ expire_in: 1 day PG12-debian10: <<: *init_PG_instance stage: build image: postgres:12 script: - apt-get update - apt-get install -y --no-install-recommends make gcc git postgresql-server-dev-12 pgxnclient python3-pip - pip3 install --upgrade pip - pip3 install --no-cache-dir -r python/requirements.txt - make extension - make install - psql -c "ALTER SYSTEM SET session_preload_libraries = 'anon'" - psql -c "SELECT pg_reload_conf();" - make installcheck || diff results tests/expected artifacts: paths: - anon* - regression.* - results/ expire_in: 1 day PG13-debian10: <<: *init_PG_instance stage: build image: postgres:13 script: - apt-get update - apt-get install -y --no-install-recommends make gcc postgresql-server-dev-13 python3-pip diffutils - pip3 install --upgrade pip - pip3 install --no-cache-dir -r python/requirements.txt - make extension - make install - psql -c "ALTER SYSTEM SET session_preload_libraries = 'anon'" - psql -c "SELECT pg_reload_conf();" - make installcheck || diff results tests/expected artifacts: paths: - anon* - regression.* - results/ expire_in: 1 day PG14-rocky8: stage: build image: rockylinux/rockylinux:8 script: - dnf -y install https://download.postgresql.org/pub/repos/yum/reporpms/EL-8-x86_64/pgdg-redhat-repo-latest.noarch.rpm - dnf -qy module disable postgresql # PG14 is still in beta - dnf -y install 'dnf-command(config-manager)' - dnf config-manager --enable pgdg14-updates-testing - dnf -y install git make gcc redhat-rpm-config postgresql14 postgresql14-server postgresql14-contrib postgresql14-devel diffutils # Package python3-faker-6.1.1-1.rhel8.noarch.rpm is not signed - dnf -y --nogpgcheck install python3 python3-faker python3-dateutil - export PATH="$PATH:/usr/pgsql-14/bin" # python3-faker requires python3-text-unidecode in EPEL - dnf -y install epel-release - dnf -y install python3-text-unidecode - make - make install - mkdir -p $PGDATA - chown postgres $PGDATA - su postgres -c 'initdb' - su postgres -c 'pg_ctl start' - psql -c "ALTER SYSTEM SET session_preload_libraries = 'anon'" - psql -c "SELECT pg_reload_conf();" - make installcheck || diff results tests/expected artifacts: paths: - anon* - regression.* - results/ expire_in: 1 day blackbox: stage: build image: docker:19.03.8 services: - docker:19.03.8-dind script: - docker build -t blackbox . --file docker/Dockerfile - mkdir results - cat tests/sql/blackbox.sql | docker run --rm -i blackbox /anon.sh > results/blackbox.out # we remove comments because pg_dump ouputs the PG version and we don't want # this test to break every time a new minor version is released - sed -i 's/^--.*$//' results/blackbox.out - diff tests/expected/blackbox.out results/blackbox.out ## ## T E S T ## ## automatic SAST from Gitlab.com include: - template: Security/SAST.gitlab-ci.yml ## basic user sories demo: <<: *init_PG_instance stage: test script: - apt-get update - apt-get install -y --no-install-recommends make gcc postgresql-server-dev-13 postgresql-contrib-13 pgxnclient - pgxn install ddlx - make extension - make install - psql -c "ALTER SYSTEM SET session_preload_libraries = 'anon'" - psql -c "SELECT pg_reload_conf();" - make demo when: manual artifacts: paths: - demo/*.out expire_in: 1 day ## Test various installation process install_pgxn: stage: test script: - apt-get update - apt-get install -y --no-install-recommends make gcc postgresql-server-dev-13 pgxnclient - pgxn install postgresql_anonymizer when: manual install_pgxn_ubuntu_pg95: stage: test image: ubuntu:bionic script: - apt-get update - apt-get install -y --no-install-recommends make gcc postgresql-common pgxnclient gnupg ca-cacert - yes '' | sh /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh bionic - apt-get install -y --no-install-recommends postgres-9.5 postgres-9.5-contrib postgresql-server-dev-9.5 - export PGDATA=/var/lib/postgresql/data - export PGUSER=postgres - su postgres -c /usr/lib/postgresql/9.5/bin/initdb - su postgres -c "/usr/lib/postgresql/9.5/bin/pg_ctl start" & - pg_config --sharedir - pgxn install --pg_config /usr/lib/postgresql/9.5/bin/pg_config postgresql_anonymizer - psql -c "CREATE EXTENSION tsm_system_rows;" - psql -c "CREATE TEMPORARY TABLE pg_config AS SELECT 'SHAREDIR'::TEXT AS name, '/usr/share/postgresql/9.5'::TEXT AS setting;CREATE EXTENSION anon;" - psql -c "ALTER DATABASE postgres SET session_preload_libraries='anon'" - psql -c "SELECT anon.init();" when: manual install_yum_centos7_pg12: stage: test image: centos:7 script: - yum -y install https://download.postgresql.org/pub/repos/yum/reporpms/EL-7-x86_64/pgdg-redhat-repo-latest.noarch.rpm - yum -y install postgresql12-contrib postgresql_anonymizer12 - mkdir -p $PGDATA - chown postgres $PGDATA - su postgres -c /usr/pgsql-12/bin/initdb $PGDATA - su postgres -c "/usr/pgsql-12/bin/pg_ctl start" - su postgres -c "psql -c 'CREATE EXTENSION anon CASCADE;'" when: manual install_source_fedora: stage: test image: fedora:34 script: - dnf install --assumeyes postgresql-server postgresql-contrib postgresql-server-devel libpq-devel make git redhat-rpm-config diffutils python3-faker - cd /tmp - git clone https://gitlab.com/dalibo/postgresql_anonymizer.git - cd postgresql_anonymizer - make - make install - export PGDATA=/var/lib/postgresql/data - mkdir -p $PGDATA - chown postgres $PGDATA - su postgres -c initdb - su postgres -c "pg_ctl start" - export PGUSER=postgres - psql -c "ALTER SYSTEM SET session_preload_libraries = 'anon'" - psql -c "SELECT pg_reload_conf();" - make installcheck || diff results tests/expected when: manual ## ## D E P L O Y ## docker_latest: stage: deploy image: docker:19.03.8 services: - docker:19.03.8-dind script: - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY - docker build -t $CI_REGISTRY/dalibo/postgresql_anonymizer:latest . --file docker/Dockerfile - docker push $CI_REGISTRY/dalibo/postgresql_anonymizer:latest only: - master@dalibo/postgresql_anonymizer docker_stable: stage: deploy image: docker:19.03.8 services: - docker:19.03.8-dind script: - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY - docker build -t $CI_REGISTRY/dalibo/postgresql_anonymizer:stable . --file docker/Dockerfile - docker push $CI_REGISTRY/dalibo/postgresql_anonymizer:stable only: - stable@dalibo/postgresql_anonymizer pgxn: stage: deploy image: alpine script: - apk add -U make git zip - make pgxn artifacts: paths: - anon* - _pgxn/ expire_in: 1 day only: - master