Feature/external deployment replication

jobs/pxc-mysql/spec

@@ -51,6 +51,14 @@ consumes:
   optional: true
 
 provides:
+- name: replica
+  type: conn
+  properties:
+  - port
+  - mysql_version
+  - mysql_backup_username
+  - mysql_backup_password
+
 - name: mysql
   type: mysql
   properties:
@@ -67,15 +75,13 @@ provides:
   - mysql_socket
 
 properties:
-
   pxc_enabled:
     description: 'Used for disabling the job. Useful if co-locating the cf-mysql release mysql job and migrating'
     default: true
   disable_persistent_storage_safety_check:
     description: 'pre-start checks that /var/vcap/store is a persistent volume to prevent accidentally running the database on an ephemeral disk. This can be used to disable this check for test or bosh-lite situations'
     default: false
 
-
   # Admin Users
   admin_username:
     description: 'Username for the MySQL server admin user'
@@ -309,6 +315,12 @@ properties:
   engine_config.read_write_permissions:
     description: "Specify the database server's read/write setting. For single-node deployments, valid options are `read_write`, `read_only`, or `super_read_only`. The setting must be `read_write` for Galera clusters."
     default: read_write
+  engine_config.enable_replication_target:
+    description: 'When configuring a replica, this will use spec.index to populate server_id on each node starting at the highest possible index. Any value set for server_id will be discarded by enabling this setting'
+    default: false
+  engine_config.enable_replication_source:
+    description: 'When configuring a replica, this will use spec.index to populate server_id on each node starting at the lowest possible index. Any value set for server_id will be discarded by enabling this setting'
+    default: false
   engine_config.server_id:
     description: 'In leader-follower topology, this value must be unique. In other words, the leader must have a different value than the follower and vice versa. If this is set to 0, then the server refuses any replication connections.'
     default: 0
@@ -388,4 +400,4 @@ properties:
   kernel.vm.swappiness:
     description: "Configure Linux vm.swappiness"
     # https://docs.kernel.org/admin-guide/sysctl/vm.html#swappiness
-    # https://www.percona.com/blog/mysql-101-linux-tuning-for-mysql/
+    # https://www.percona.com/blog/mysql-101-linux-tuning-for-mysql/

jobs/pxc-mysql/templates/db_init.erb

@@ -22,13 +22,19 @@ DROP USER IF EXISTS '<%= p('admin_username') %>'@'<%= host %>';
   <%- end -%>
 DROP USER IF EXISTS 'roadmin'@'<%= host %>';
 
-<%- end -%>
-<%- if_p('mysql_backup_password') do |password| -%>
-CREATE USER IF NOT EXISTS '<%= p('mysql_backup_username') %>'@'localhost';
-ALTER USER '<%= p('mysql_backup_username') %>'@'localhost' IDENTIFIED WITH <%= p('engine_config.user_authentication_policy') %> BY '<%= password %>';
-GRANT RELOAD, LOCK TABLES, REPLICATION SLAVE, REPLICATION CLIENT, /*!80001 BACKUP_ADMIN,*/ PROCESS ON *.* to '<%= p('mysql_backup_username') %>'@'localhost';
-GRANT SELECT on performance_schema.keyring_component_status to '<%= p('mysql_backup_username') %>'@'localhost';
-GRANT SELECT ON performance_schema.log_status TO '<%= p('mysql_backup_username') %>'@'localhost';
+<%- end
+allowed_remote_backup_hosts='localhost'
+
+if p('engine_config.enable_replication_source')
+  allowed_remote_backup_hosts='%'
+end
+
+if_p('mysql_backup_password') do |password| -%>
+CREATE USER IF NOT EXISTS '<%= p('mysql_backup_username') %>'@'<%= allowed_remote_backup_hosts %>';
+ALTER USER '<%= p('mysql_backup_username') %>'@'<%= allowed_remote_backup_hosts %>' IDENTIFIED WITH <%= p('engine_config.user_authentication_policy') %> BY '<%= password %>';
+GRANT RELOAD, LOCK TABLES, REPLICATION SLAVE, REPLICATION CLIENT, /*!80001 BACKUP_ADMIN,*/ PROCESS ON *.* to '<%= p('mysql_backup_username') %>'@'<%= allowed_remote_backup_hosts %>';
+GRANT SELECT on performance_schema.keyring_component_status to '<%= p('mysql_backup_username') %>'@'<%= allowed_remote_backup_hosts %>';
+GRANT SELECT ON performance_schema.log_status TO '<%= p('mysql_backup_username') %>'@'<%= allowed_remote_backup_hosts %>';
 
 <%- end -%>
 <%- hosts.each do |host| -%>

jobs/pxc-mysql/templates/my.cnf.erb

@@ -60,7 +60,22 @@ character_set_server            = <%= p('engine_config.character_set_server') %>
 <%- if p('engine_config.collation_server') != 'use_default' -%>
 collation_server                = <%= p('engine_config.collation_server') %>
 <%- end -%>
-server_id                       = <%= p('engine_config.server_id') %>
+
+<%
+server_id = p('engine_config.server_id')
+# ensure we're not starting at index 0. replication is disabled when it is set to `0`
+# it just needs to be unique
+# https://dev.mysql.com/doc/refman/8.4/en/replication-options.html
+if p('engine_config.enable_replication_source')
+  server_id = spec.index + 1
+end
+# if where the replication target, count down from highest value to avoid overlaps between source and target
+if p('engine_config.enable_replication_target')
+  server_id = 4294967296 - spec.index - 1 - 1 # max value is `2^32 - 1`. spec.index can be 0.
+end
+%>
+server_id                       = <%= server_id %>
+
 <%- if_p('tls.client.ca') do -%>
 ssl_ca=/var/vcap/jobs/pxc-mysql/certificates/client-ca.pem
 <%- end -%>

jobs/pxc-replicator/monit

@@ -0,0 +1,7 @@
+<% if p('replicator_enabled') == true %>
+check process pxc-replicator
+  with pidfile /var/vcap/sys/run/bpm/pxc-replicator/pxc-replicator.pid
+  start program "/var/vcap/jobs/bpm/bin/bpm start pxc-replicator" with timeout <%= p('monit_startup_timeout') %> seconds
+  stop program "/var/vcap/jobs/bpm/bin/bpm stop pxc-replicator"
+  group vcap
+<% end %>

jobs/pxc-replicator/spec

@@ -0,0 +1,58 @@
+---
+name: pxc-replicator
+
+templates:
+  bpm.yml.erb: config/bpm.yml
+  bin/setup: bin/setup
+  bin/post-start: bin/post-start
+  source.mysql.cnf.erb: config/source.mysql.cnf
+
+packages:
+- percona-xtradb-cluster-8.0
+- percona-xtradb-cluster-8.4
+- percona-xtrabackup-2.4
+- percona-xtrabackup-8.0
+- percona-xtrabackup-8.4
+- pxc-utils
+
+consumes:
+- name: replica
+  type: conn
+  properties:
+  - port
+  - mysql_version
+  - mysql_backup_username
+  - mysql_backup_password
+  - tls
+
+properties:
+  monit_startup_timeout:
+    description: 'How long to wait for monit to show running for the process'
+    default: 5
+  mysql_version:
+    description: 'deployed version'
+    default: 8.0
+  logging.format.timestamp:
+    default: rfc3339
+  replicator_enabled:
+    description: 'Whether to enable the job ( skips writing monit file if set to false )'
+    default: true
+  config.source.tls:
+    description: 'TLS certificates to use for connections'
+    example: {
+      "ca": "...",
+      "certificate": "...",
+      "private_key": "..."
+    }
+    default: {}
+  config.source.tls:
+    description: 'TLS certificates to use for connections'
+    example: {
+      "ca": "...",
+      "certificate": "...",
+      "private_key": "..."
+    }
+    default: {}
+  config.tls.enabled:
+    description: 'Use TLS for connections & authentication'
+    default: false

jobs/pxc-replicator/templates/bin/post-start

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+. /var/vcap/packages/pxc-utils/logging.sh
+
+#it's a bash feature.
+#https://www.gnu.org/software/bash/manual/html_node/Bash-Variables.html#index-SECONDS
+
+SECONDS=0
+
+while [[ $SECONDS -lt <%= p('monit_startup_timeout') %> ]]; do
+  if grep "$(date +%Y-%m-%dT%H:%M).*replication healthy" /var/vcap/sys/log/pxc-replicator/pxc-replicator.stdout.log > /dev/null; then
+    log "replication reported healthy within the laset minute"
+    exit 0
+  fi
+  sleep 5
+done
+
+log "timed out waiting for replication to report healthy"
+exit -1
+

jobs/pxc-replicator/templates/bin/setup

@@ -0,0 +1,90 @@
+#!/usr/bin/env bash
+set -e
+
+#shellcheck disable=1091
+. /var/vcap/packages/pxc-utils/logging.sh
+
+function is_replication_configured(){
+
+  ! test "$($self_mysql <<< "SHOW REPLICA STATUS\G")" = ""
+}
+function is_replica_io_running (){
+  $self_mysql <<< "SHOW REPLICA STATUS\G" | grep 'Replica_IO_Running: Yes'
+}
+function is_replica_sql_running (){
+  $self_mysql <<< "SHOW REPLICA STATUS\G" | grep 'Replica_SQL_Running: Yes'
+}
+function enable_replication (){
+  $self_mysql <<< "STOP REPLICA;
+CHANGE REPLICATION SOURCE TO
+    SOURCE_HOST='${SOURCE_ADDR}',
+    SOURCE_USER='${SOURCE_USER}',
+    SOURCE_PASSWORD='${SOURCE_PASS}',
+    SOURCE_AUTO_POSITION=1;
+START REPLICA;"
+}
+function sync_databases(){
+  log "resyncing with full backup"
+  log $($self_mysql <<< "STOP REPLICA;")
+  log $($self_mysql <<< "RESET REPLICA;")
+  log $($self_mysql <<< "RESET MASTER;")
+  log $($source_dump --all-databases --triggers --routines  --single-transaction | $self_mysql)
+}
+function source_upcheck(){
+  $source_mysql <<< "exit"
+}
+function self_upcheck(){
+  $self_mysql <<< "exit"
+}
+
+while ! source_upcheck; do
+  log "waiting for source to be available"
+  sleep 5
+done
+
+while ! self_upcheck; do
+  log "waiting for self to be available"
+  sleep 5
+done
+
+log "starting replication setup"
+
+log "checking existing databases in source instance"
+
+log "checking replication status"
+
+if [[ ! is_replication_configured ]]; then
+  log "replication is not yet enabled"
+  sync_databases
+
+  log "replication is not configured. enabling"
+  OUT=$(enable_replication);
+  if [[ $(enable_replication) !=  "" ]]; then
+    log "failed enabling replication: '$OUT'" | sed "s/${PASS}/<REDACTED>/g"
+  else
+    log "replication enabled"
+  fi
+else
+  log "replicaton already enabled, skipping"
+fi
+
+while true; do
+  log "checking running status"
+  RUNNING_STATUS="$($self_mysql <<< "SHOW REPLICA STATUS\G" | grep 'Replica_.*_Running: ' | xargs )"
+  if is_replica_io_running && is_replica_sql_running; then
+    log "replication healthy"
+  elif ! is_replica_io_running; then
+    log "restarting replica"
+    $self_mysql <<< "START REPLICA;"
+  elif ! is_replica_sql_running ; then
+    log "replication Replica_SQL_Running marked as no. Attempting to resync"
+    sync_databases
+  else
+    log "$RUNNING_STATUS"
+    for line in $($self_mysql <<< "SHOW REPLICA STATUS\G" | grep 'Err' | tr -d '[:blank:]' ); do
+      log "${line/$SOURCE_PASS/<redacted>/}";
+    done
+  fi
+  log "$RUNNING_STATUS"
+  sleep 5
+done

jobs/pxc-replicator/templates/bpm.yml.erb

@@ -0,0 +1,34 @@
+<%-
+if !['rfc3339', 'unix-epoch'].include?(p('logging.format.timestamp'))
+  raise "'#{p('logging.format.timestamp')}' is not a valid timestamp format for the property 'logging.format.timestamp'." +
+        " Valid options are: 'rfc3339' and 'unix-epoch'."
+end
+path = [
+  "/usr/bin",
+  "/bin",
+  "/var/vcap/packages/percona-xtradb-cluster-#{p('mysql_version')}/bin",
+]
+-%>
+---
+processes:
+- name: pxc-replicator
+  executable: /var/vcap/jobs/pxc-replicator/bin/setup
+  args: []
+  env:
+    PATH: <%= path.join(":") %>
+    SOURCE_USER: <%= link('replica').p('mysql_backup_username') %>
+    SOURCE_PASS: <%= link('replica').p('mysql_backup_password') %>
+    SOURCE_ADDR: <%= link('replica').address %>
+    self_mysql: 'mysql --defaults-file=/var/vcap/jobs/pxc-mysql/config/mylogin.cnf'
+    source_dump: 'mysqldump --defaults-file=/var/vcap/jobs/pxc-replicator/config/source.mysql.cnf'
+    source_mysql: 'mysql --defaults-file=/var/vcap/jobs/pxc-replicator/config/source.mysql.cnf'
+
+  persistent_disk: true
+  ephemeral_disk: true
+  additional_volumes:
+  - path: /var/vcap/packages/pxc-utils
+    writeable: false
+  - path: /var/vcap/sys/run/pxc-mysql
+    writeable: false
+  - path: /var/vcap/jobs/pxc-mysql/config
+  - path: /var/vcap/jobs/pxc-mysql/certificates

jobs/pxc-replicator/templates/source.mysql.cnf.erb

@@ -0,0 +1,11 @@
+<%-
+require 'json'
+if_link('replica') do |source| %>
+[client]
+user = <%= source.p('mysql_backup_username') %>
+password = <%= source.p('mysql_backup_password') %>
+host = <%= source.instances[0].address %>
+port = <%= source.p('port') %>
+<%- end  %>
+
+

operations/add-replicator-job.yml

@@ -0,0 +1,11 @@
+- type: replace
+  path: /instance_groups/name=mysql/jobs/name=pxc-replicator?
+  value:
+    name: pxc-replicator
+    release: pxc
+    properties:
+      config.tls.enabled: true
+
+- type: replace
+  path: /instance_groups/name=mysql/jobs/name=pxc-mysql/properties/engine_config?/binlog?/enable_gtid_mode?
+  value: true

operations/local-dev-release.yml

@@ -0,0 +1,6 @@
+- type: replace
+  path: /releases/name=pxc
+  value:
+    name: pxc
+    version: create
+    url: file://./

operations/set-replicator-source.yml

@@ -0,0 +1,23 @@
+- type: replace
+  path: /instance_groups/name=mysql/jobs/name=pxc-mysql/properties/engine_config?/enable_replication_source
+  value: true
+- type: replace
+  path: /instance_groups/name=mysql/jobs/name=pxc-mysql/provides?/replica
+  value:
+    shared: true
+    as: source
+- type: replace
+  path: /instance_groups/name=mysql/jobs/name=pxc-mysql/properties/mysql_backup_username?
+  value: ((mysql_backup_user.username))
+- type: replace
+  path: /instance_groups/name=mysql/jobs/name=pxc-mysql/properties/mysql_backup_password?
+  value: ((mysql_backup_user.password))
+- type: replace
+  path: /variables/name=mysql_backup_user?
+  value:
+    name: mysql_backup_user
+    type: user
+    parameters:
+      length: 32
+
+

operations/set-replicator-target.yml

@@ -0,0 +1,8 @@
+- type: replace
+  path: /instance_groups/name=mysql/jobs/name=pxc-mysql/properties/engine_config?/enable_replication_target
+  value: true
+- type: replace
+  path: /instance_groups/name=mysql/jobs/name=pxc-replicator/consumes?/replica
+  value:
+    from: source
+    deployment: ((source_deployment_name))

operations/update-azs.yml

@@ -0,0 +1,3 @@
+- type: replace
+  path: /instance_groups/name=mysql/azs?
+  value: ((azs))

operations/update-deployment-name.yml

@@ -0,0 +1,3 @@
+- type: replace
+  path: /name
+  value: ((deployment_name))

operations/update-network.yml

@@ -0,0 +1,3 @@
+- type: replace
+  path: /instance_groups/name=mysql/networks?
+  value: [{name: ((network_name))}]

operations/update-persistent-disk-size.yml

@@ -0,0 +1,3 @@
+- type: replace
+  path: /instance_groups/name=mysql/vm_type?
+  value: ((vm_type))

operations/update-vm-type.yml

@@ -0,0 +1,3 @@
+- type: replace
+  path: /instance_groups/name=mysql/vm_type?
+  value: ((vm_type))