Create PSQL_MODE master, slave and snapshot for pg_basebackup and streaming replication

2026-06-27 23:22:06 +08:00 · 2015-06-16 19:15:01 -04:00
parent 209a5f095a
commit 9dd5b55b3a
2 changed files with 200 additions and 40 deletions
@@ -9,6 +9,7 @@
 - [Installation](#installation)
 - [Quick Start](#quick-start)
 - [Creating User and Database at Launch](#creating-user-and-database-at-launch)
+- [Creating a Snapshot or Slave Database](#creating-a-snapshot-or-slave-database)
 - [Configuration](#configuration)
    - [Data Store](#data-store)
 - [Shell Access](#shell-access)
@@ -143,6 +144,68 @@ This has the effect of adding the following to the `pg_hba.conf` file:
 host    all             all             samenet                 trust
 ```

+# Creating a Snapshot or Slave Database
+
+You may use the `PSQL_MODE` variable along with `REPLICATION_HOST`, `REPLICATION_PORT`, `REPLICATION_USER` and `REPLICATION_PASS` to create a snapshot of an existing database and enable stream replication.
+
+Your master database must support replication or super-user access for the credentials you specify. The `PSQL_MODE` variable should be set to "master" for replication on your master node and "slave" or "snapshot" for streaming replication or a point-in-time snapshot of a running database.
+
+Create a new master instance:
+
+```bash
+docker run --name postgresql-master 5432:5432 -d \
+  -e 'PSQL_TRUST_LOCALNET=true' \
+  -e 'PSQL_MODE=master' \
+  -e 'DB_NAME=dbname' \
+  -e 'DB_USER=dbuser' -e 'DB_PASS=dbpass' \
+  -e 'REPLICATION_USER=replicator'  -e 'REPLICATION_PASS=replicatorpass' \
+  sameersbn/postgresql:9.4
+```
+
+Create a slave instance with streaming replication:
+
+```bash
+docker run --name postgresql-slave -p 5433:5432 -d \
+  -e 'PSQL_TRUST_LOCALNET=true' \
+  -e 'PSQL_MODE=slave' \
+  -e 'REPLICATION_HOST=localhost' -e 'REPLICATION_PORT=5432' \
+  -e 'REPLICATION_USER=replicator' -e 'REPLICATION_PASS=replicatorpass' \
+  sameersbn/postgresql:9.4
+```
+
+Create a snapshot downloaded from a master instance:
+```bash
+docker run --name postgresql-slave -p 5433:5432 -d \
+  -e 'PSQL_TRUST_LOCALNET=true' \
+  -e 'PSQL_MODE=snapshot' \
+  -e 'REPLICATION_HOST=localhost' -e 'REPLICATION_PORT=5432' \
+  -e 'REPLICATION_USER=replicator' -e 'REPLICATION_PASS=replicatorpass' \
+  sameersbn/postgresql:9.4
+```
+
+You may use docker links and interactive shells when testing this module:
+
+```bash
+mkdir -p /tmp/postgresql-master
+mkdir -p /tmp/postgresql-slave
+# master
+docker run --name='postgresql-master' -it --rm \
+  --volume=/tmp/postgresql-master:/var/lib/postgresql \
+  -e 'PSQL_TRUST_LOCALNET=true' \
+  -e 'PSQL_MODE=master' \
+  -e 'REPLICATION_USER=replicator'  -e 'REPLICATION_PASS=replicatorpass' \
+  sameersbn/postgresql:latest
+
+# slave
+docker run --link postgresql-master:psql --name='postgresql-slave' -it --rm \
+  --volume=/tmp/postgresql-slave:/var/lib/postgresql \
+  -e 'PSQL_TRUST_LOCALNET=true' \
+  -e 'PSQL_MODE=slave' \
+  -e 'REPLICATION_HOST=psql' -e 'REPLICATION_USER=replicator' -e 'REPLICATION_PASS=replicatorpass' \
+  sameersbn/postgresql:latest
+```
+
+
 # Configuration

 ## Data Store
@@ -5,6 +5,7 @@ PG_HOME="/var/lib/postgresql"
 PG_CONFDIR="/etc/postgresql/${PG_VERSION}/main"
 PG_BINDIR="/usr/lib/postgresql/${PG_VERSION}/bin"
 PG_DATADIR="${PG_HOME}/${PG_VERSION}/main"
+PG_ARCHIVEDIR="${PG_HOME}/${PG_VERSION}/archive"

 if [ -n "${USERMAP_UID}" ] || [ -n "${USERMAP_GID}" ]; then
  if [ -n "${USERMAP_UID}" ] && [ -n "${USERMAP_GID}" ]; then
@@ -21,13 +22,29 @@ fi
 # set this env variable to true to enable a line in the
 # pg_hba.conf file to trust samenet.  this can be used to connect
 # from other containers on the same host without authentication
-PSQL_TRUST_LOCALNET=${PSQL_TRUST_LOCALNET:false}
+PSQL_TRUST_LOCALNET=${PSQL_TRUST_LOCALNET:-false}

 DB_NAME=${DB_NAME:-}
 DB_USER=${DB_USER:-}
 DB_PASS=${DB_PASS:-}
 DB_UNACCENT=${DB_UNACCENT:false}

+# by default postgresql will start up as a standalone instance.
+# set this environment variable to master, slave or snapshot to use replication features.
+# "snapshot" will create a point in time backup of a master instance.
+PSQL_MODE=${PSQL_MODE:-"standalone"}
+
+# postgresql wal archives are used for point-in-time recovery and delayed replication.
+PSQL_ARCHIVEMODE=${PSQL_ARCHIVEMODE:-"off"}
+
+REPLICATION_USER=${REPLICATION_USER:-}
+REPLICATION_PASS=${REPLICATION_PASS:-}
+REPLICATION_HOST=${REPLICATION_HOST:-}
+REPLICATION_PORT=${REPLICATION_PORT:-5432}
+
+# set this env variable to "require" to enable encryption and "verify-full" for verification.
+PSQL_SSLMODE=${PSQL_SSLMODE:-"disable"}
+
 # fix ownership of ${PG_CONFDIR} (may be necessary if USERMAP_* was set)
 chown -R postgres:postgres ${PG_CONFDIR}

@@ -40,8 +57,9 @@ mkdir -p -m 0755 /run/postgresql /run/postgresql/${PG_VERSION}-main.pg_stat_tmp
 chown -R postgres:postgres /run/postgresql
 chmod g+s /run/postgresql

-# disable ssl
-sed 's/ssl = true/#ssl = true/' -i ${PG_CONFDIR}/postgresql.conf
+if [ "${PSQL_SSLMODE}" == "disable" ]; then
+  sed 's/ssl = true/#ssl = true/' -i ${PG_CONFDIR}/postgresql.conf
+fi

 # listen on all interfaces
 cat >> ${PG_CONFDIR}/postgresql.conf <<EOF
@@ -60,18 +78,78 @@ cat >> ${PG_CONFDIR}/pg_hba.conf <<EOF
 host    all             all             0.0.0.0/0               md5
 EOF

+# allow replication connections to the database
+if [ -n "${REPLICATION_USER}" ]; then
+  if [ "${PSQL_SSLMODE}" == "disable" ]; then
+    cat >> ${PG_CONFDIR}/pg_hba.conf <<EOF
+host    replication     $REPLICATION_USER       0.0.0.0/0               md5
+EOF
+  else
+    cat >> ${PG_CONFDIR}/pg_hba.conf <<EOF
+hostssl replication     $REPLICATION_USER       0.0.0.0/0               md5
+EOF
+  fi
+fi
+
+if [ "${PSQL_MODE}" == "master" ]; then
+  if [ -n "${REPLICATION_USER}" ]; then
+    echo "Supporting hot standby..."
+    cat >> ${PG_CONFDIR}/postgresql.conf <<EOF
+wal_level = hot_standby
+max_wal_senders = 3
+checkpoint_segments = 8
+wal_keep_segments = 8
+EOF
+    if [ "${PSQL_ARCHIVEMODE}" == "on" ]; then
+      sudo -u postgres mkdir -p ${PG_ARCHIVEDIR}
+      cat >> ${PG_CONFDIR}/postgresql.conf <<EOF
+archive_mode = ${PSQL_ARCHIVEMODE}
+archive_command = 'test ! -f ${PG_ARCHIVEDIR}/%f && cp %p ${PG_ARCHIVEDIR}/%f'
+archive_cleanup_command = 'pg_archivecleanup ${PSQL_ARCHIVEMODE} %r'
+EOF
+    fi
+  fi
+fi
+
+
 cd ${PG_HOME}

 # initialize PostgreSQL data directory
 if [ ! -d ${PG_DATADIR} ]; then
-  # check if we need to perform data migration
-  PG_OLD_VERSION=$(find ${PG_HOME}/[0-9].[0-9]/main -maxdepth 1 -name PG_VERSION 2>/dev/null | sort -r | head -n1 | cut -d'/' -f5)
+  if [ "${PSQL_MODE}" == "slave" ] || [ "${PSQL_MODE}" == "snapshot" ]; then
+    echo "Replicating database..."
+    if [ "${PSQL_MODE}" == "snapshot" ]; then
+      sudo -u postgres -H \
+        PGPASSWORD=$REPLICATION_PASS "${PG_BINDIR}/pg_basebackup" -D "${PG_DATADIR}" \
+        -h "${REPLICATION_HOST}" -p "${REPLICATION_PORT}" -U "${REPLICATION_USER}" -w -x -v -P
+    elif [ "${PSQL_MODE}" == "slave" ]; then
+      # Setup streaming replication.
+      sudo -u postgres -H \
+        PGPASSWORD=$REPLICATION_PASS "${PG_BINDIR}/pg_basebackup" -D "${PG_DATADIR}" \
+        -h "${REPLICATION_HOST}" -p "${REPLICATION_PORT}" -U "${REPLICATION_USER}" -w -v -P
+      echo "Setting up hot standby configuration..."
+      cat >> ${PG_CONFDIR}/postgresql.conf <<EOF
+hot_standby = on
+EOF
+      sudo -u postgres touch ${PG_DATADIR}/recovery.conf
+      cat >> ${PG_DATADIR}/recovery.conf <<EOF
+standby_mode = 'on'
+primary_conninfo = 'host=${REPLICATION_HOST} port=${REPLICATION_PORT} user=${REPLICATION_USER} password=${REPLICATION_PASS} sslmode=${PSQL_SSLMODE}'
+trigger_file = '/tmp/postgresql.trigger'
+EOF
+    fi

-  echo "Initializing database..."
-  sudo -u postgres -H "${PG_BINDIR}/initdb" --pgdata="${PG_DATADIR}" \
-    --username=postgres --encoding=unicode --auth=trust >/dev/null
+  else
+    # check if we need to perform data migration
+    PG_OLD_VERSION=$(find ${PG_HOME}/[0-9].[0-9]/main -maxdepth 1 -name PG_VERSION 2>/dev/null | sort -r | head -n1 | cut -d'/' -f5)
+
+    echo "Initializing database..."
+    sudo -u postgres -H "${PG_BINDIR}/initdb" --pgdata="${PG_DATADIR}" \
+      --username=postgres --encoding=unicode --auth=trust >/dev/null
+  fi
 fi

+
 if [ -n "${PG_OLD_VERSION}" ]; then
  echo "Migrating postgresql ${PG_OLD_VERSION} data..."
  PG_OLD_CONFDIR="/etc/postgresql/${PG_OLD_VERSION}/main"
@@ -97,42 +175,61 @@ if [ -n "${PG_OLD_VERSION}" ]; then
    -O "-c config_file=${PG_CONFDIR}/postgresql.conf" >/dev/null
 fi

-if [ -n "${DB_USER}" ]; then
-  if [ -z "${DB_PASS}" ]; then
-    echo ""
-    echo "WARNING: "
-    echo "  Please specify a password for \"${DB_USER}\". Skipping user creation..."
-    echo ""
-    DB_USER=
-  else
-    echo "Creating user \"${DB_USER}\"..."
-    echo "CREATE ROLE ${DB_USER} with LOGIN CREATEDB PASSWORD '${DB_PASS}';" |
-      sudo -u postgres -H ${PG_BINDIR}/postgres --single \
-        -D ${PG_DATADIR} -c config_file=${PG_CONFDIR}/postgresql.conf >/dev/null
-  fi
-fi

-if [ -n "${DB_NAME}" ]; then
-  for db in $(awk -F',' '{for (i = 1 ; i <= NF ; i++) print $i}' <<< "${DB_NAME}"); do
-    echo "Creating database \"${db}\"..."
-    echo "CREATE DATABASE ${db};" | \
-      sudo -u postgres -H ${PG_BINDIR}/postgres --single \
-        -D ${PG_DATADIR} -c config_file=${PG_CONFDIR}/postgresql.conf >/dev/null
-
-    if [ "${DB_UNACCENT}" == "true" ]; then
-      echo "Installing unaccent extension..."
-      echo "CREATE EXTENSION IF NOT EXISTS unaccent;" | \
-        sudo -u postgres -H ${PG_BINDIR}/postgres --single ${db} \
-          -D ${PG_DATADIR} -c config_file=${PG_CONFDIR}/postgresql.conf >/dev/null
-    fi
-
-    if [ -n "${DB_USER}" ]; then
-      echo "Granting access to database \"${db}\" for user \"${DB_USER}\"..."
-      echo "GRANT ALL PRIVILEGES ON DATABASE ${db} to ${DB_USER};" |
+# Hot standby (slave and snapshot) servers can ignore the following code.
+if [ "${PSQL_MODE}" == "standalone" ] || [ "${PSQL_MODE}" == "master" ]; then
+  if [ -n "${REPLICATION_USER}" ]; then
+    if [ -z "${REPLICATION_PASS}" ]; then
+      echo ""
+      echo "WARNING: "
+      echo "  Please specify a password for replication user \"${REPLICATION_USER}\". Skipping user creation..."
+      echo ""
+      DB_USER=
+    else
+      echo "Creating user \"${REPLICATION_USER}\"..."
+      echo "CREATE ROLE ${REPLICATION_USER} WITH REPLICATION LOGIN ENCRYPTED PASSWORD '${REPLICATION_PASS}';" |
        sudo -u postgres -H ${PG_BINDIR}/postgres --single \
          -D ${PG_DATADIR} -c config_file=${PG_CONFDIR}/postgresql.conf >/dev/null
    fi
-  done
+  fi
+
+  if [ -n "${DB_USER}" ]; then
+    if [ -z "${DB_PASS}" ]; then
+      echo ""
+      echo "WARNING: "
+      echo "  Please specify a password for \"${DB_USER}\". Skipping user creation..."
+      echo ""
+      DB_USER=
+    else
+      echo "Creating user \"${DB_USER}\"..."
+      echo "CREATE ROLE ${DB_USER} with LOGIN CREATEDB PASSWORD '${DB_PASS}';" |
+        sudo -u postgres -H ${PG_BINDIR}/postgres --single \
+          -D ${PG_DATADIR} -c config_file=${PG_CONFDIR}/postgresql.conf >/dev/null
+    fi
+  fi
+
+  if [ -n "${DB_NAME}" ]; then
+    for db in $(awk -F',' '{for (i = 1 ; i <= NF ; i++) print $i}' <<< "${DB_NAME}"); do
+      echo "Creating database \"${db}\"..."
+      echo "CREATE DATABASE ${db};" | \
+        sudo -u postgres -H ${PG_BINDIR}/postgres --single \
+          -D ${PG_DATADIR} -c config_file=${PG_CONFDIR}/postgresql.conf >/dev/null
+
+      if [ "${DB_UNACCENT}" == "true" ]; then
+        echo "Installing unaccent extension..."
+        echo "CREATE EXTENSION IF NOT EXISTS unaccent;" | \
+          sudo -u postgres -H ${PG_BINDIR}/postgres --single ${db} \
+            -D ${PG_DATADIR} -c config_file=${PG_CONFDIR}/postgresql.conf >/dev/null
+      fi
+
+      if [ -n "${DB_USER}" ]; then
+        echo "Granting access to database \"${db}\" for user \"${DB_USER}\"..."
+        echo "GRANT ALL PRIVILEGES ON DATABASE ${db} to ${DB_USER};" |
+          sudo -u postgres -H ${PG_BINDIR}/postgres --single \
+            -D ${PG_DATADIR} -c config_file=${PG_CONFDIR}/postgresql.conf >/dev/null
+      fi
+    done
+  fi
 fi

 echo "Starting PostgreSQL server..."