git subrepo commit (merge) mailcow/src/mailcow-dockerized

subrepo: subdir:   "mailcow/src/mailcow-dockerized"
  merged:   "32243e56"
upstream: origin:   "https://github.com/mailcow/mailcow-dockerized.git"
  branch:   "master"
  commit:   "e2b4b6f6"
git-subrepo: version:  "0.4.3"
  origin:   "???"
  commit:   "???"
Change-Id: I51e2016ef5ab88a8b0bdc08551b18f48ceef0aa5
diff --git a/mailcow/src/mailcow-dockerized/helper-scripts/_cold-standby.sh b/mailcow/src/mailcow-dockerized/helper-scripts/_cold-standby.sh
new file mode 100755
index 0000000..c4439ea
--- /dev/null
+++ b/mailcow/src/mailcow-dockerized/helper-scripts/_cold-standby.sh
@@ -0,0 +1,273 @@
+#!/usr/bin/env bash
+
+PATH=${PATH}:/opt/bin
+DATE=$(date +%Y-%m-%d_%H_%M_%S)
+export LC_ALL=C
+
+echo
+echo "If this script is run automatically by cron or a timer AND you are using block-level snapshots on your backup destination, make sure both do not run at the same time."
+echo "The snapshots of your backup destination should run AFTER the cold standby script finished to ensure consistent snapshots."
+echo
+
+function docker_garbage() {
+  IMGS_TO_DELETE=()
+
+  for container in $(grep -oP "image: \Kmailcow.+" docker-compose.yml); do
+
+    REPOSITORY=${container/:*}
+    TAG=${container/*:}
+    V_MAIN=${container/*.}
+    V_SUB=${container/*.}
+    EXISTING_TAGS=$(docker images | grep ${REPOSITORY} | awk '{ print $2 }')
+
+    for existing_tag in ${EXISTING_TAGS[@]}; do
+
+      V_MAIN_EXISTING=${existing_tag/*.}
+      V_SUB_EXISTING=${existing_tag/*.}
+
+      # Not an integer
+      [[ ! ${V_MAIN_EXISTING} =~ ^[0-9]+$ ]] && continue
+      [[ ! ${V_SUB_EXISTING} =~ ^[0-9]+$ ]] && continue
+
+      if [[ ${V_MAIN_EXISTING} == "latest" ]]; then
+        echo "Found deprecated label \"latest\" for repository ${REPOSITORY}, it should be deleted."
+        IMGS_TO_DELETE+=(${REPOSITORY}:${existing_tag})
+      elif [[ ${V_MAIN_EXISTING} -lt ${V_MAIN} ]]; then
+        echo "Found tag ${existing_tag} for ${REPOSITORY}, which is older than the current tag ${TAG} and should be deleted."
+        IMGS_TO_DELETE+=(${REPOSITORY}:${existing_tag})
+      elif [[ ${V_SUB_EXISTING} -lt ${V_SUB} ]]; then
+        echo "Found tag ${existing_tag} for ${REPOSITORY}, which is older than the current tag ${TAG} and should be deleted."
+        IMGS_TO_DELETE+=(${REPOSITORY}:${existing_tag})
+      fi
+
+    done
+
+  done
+
+  if [[ ! -z ${IMGS_TO_DELETE[*]} ]]; then
+    docker rmi ${IMGS_TO_DELETE[*]}
+  fi
+}
+
+function preflight_local_checks() {
+  if [[ -z "${REMOTE_SSH_KEY}" ]]; then
+    >&2 echo -e "\e[31mREMOTE_SSH_KEY is not set\e[0m"
+    exit 1
+  fi
+
+  if [[ ! -s "${REMOTE_SSH_KEY}" ]]; then
+    >&2 echo -e "\e[31mKeyfile ${REMOTE_SSH_KEY} is empty\e[0m"
+    exit 1
+  fi
+
+  if [[ $(stat -c "%a" "${REMOTE_SSH_KEY}") -ne 600 ]]; then
+    >&2 echo -e "\e[31mKeyfile ${REMOTE_SSH_KEY} has insecure permissions\e[0m"
+    exit 1
+  fi
+
+  if [[ ! -z "${REMOTE_SSH_PORT}" ]]; then
+    if [[ ${REMOTE_SSH_PORT} != ?(-)+([0-9]) ]] || [[ ${REMOTE_SSH_PORT} -gt 65535 ]]; then
+      >&2 echo -e "\e[31mREMOTE_SSH_PORT is set but not an integer < 65535\e[0m"
+      exit 1
+    fi
+  fi
+
+  if [[ -z "${REMOTE_SSH_HOST}" ]]; then
+    >&2 echo -e "\e[31mREMOTE_SSH_HOST cannot be empty\e[0m"
+    exit 1
+  fi
+
+  for bin in rsync docker-compose docker grep cut; do
+    if [[ -z $(which ${bin}) ]]; then
+      >&2 echo -e "\e[31mCannot find ${bin} in local PATH, exiting...\e[0m"
+      exit 1
+    fi
+  done
+
+  if grep --help 2>&1 | head -n 1 | grep -q -i "busybox"; then
+    >&2 echo -e "\e[31mBusyBox grep detected on local system, please install GNU grep\e[0m"
+    exit 1
+  fi
+}
+
+function preflight_remote_checks() {
+
+  if ! ssh -o StrictHostKeyChecking=no \
+    -i "${REMOTE_SSH_KEY}" \
+    ${REMOTE_SSH_HOST} \
+    -p ${REMOTE_SSH_PORT} \
+    rsync --version > /dev/null ; then
+      >&2 echo -e "\e[31mCould not verify connection to ${REMOTE_SSH_HOST}\e[0m"
+      >&2 echo -e "\e[31mPlease check the output above (is rsync >= 3.1.0 installed on the remote system?)\e[0m"
+      exit 1
+  fi
+
+  if ssh -o StrictHostKeyChecking=no \
+    -i "${REMOTE_SSH_KEY}" \
+    ${REMOTE_SSH_HOST} \
+    -p ${REMOTE_SSH_PORT} \
+    grep --help 2>&1 | head -n 1 | grep -q -i "busybox" ; then
+      >&2 echo -e "\e[31mBusyBox grep detected on remote system ${REMOTE_SSH_HOST}, please install GNU grep\e[0m"
+      exit 1
+  fi
+
+  for bin in rsync docker-compose docker; do
+    if ! ssh -o StrictHostKeyChecking=no \
+      -i "${REMOTE_SSH_KEY}" \
+      ${REMOTE_SSH_HOST} \
+      -p ${REMOTE_SSH_PORT} \
+      which ${bin} > /dev/null ; then
+        >&2 echo -e "\e[31mCannot find ${bin} in remote PATH, exiting...\e[0m"
+        exit 1
+    fi
+  done
+
+}
+
+preflight_local_checks
+preflight_remote_checks
+
+SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+COMPOSE_FILE="${SCRIPT_DIR}/../docker-compose.yml"
+source "${SCRIPT_DIR}/../mailcow.conf"
+CMPS_PRJ=$(echo ${COMPOSE_PROJECT_NAME} | tr -cd 'A-Za-z-_')
+SQLIMAGE=$(grep -iEo '(mysql|mariadb)\:.+' "${COMPOSE_FILE}")
+
+echo
+echo -e "\033[1mFound compose project name ${CMPS_PRJ} for ${MAILCOW_HOSTNAME}\033[0m"
+echo -e "\033[1mFound SQL ${SQLIMAGE}\033[0m"
+echo
+
+# Make sure destination exists, rsync can fail under some circumstances
+echo -e "\033[1mPreparing remote...\033[0m"
+if ! ssh -o StrictHostKeyChecking=no \
+  -i "${REMOTE_SSH_KEY}" \
+  ${REMOTE_SSH_HOST} \
+  -p ${REMOTE_SSH_PORT} \
+  mkdir -p "${SCRIPT_DIR}/../" ; then
+    >&2 echo -e "\e[31m[ERR]\e[0m - Could not prepare remote for mailcow base directory transfer"
+    exit 1
+fi
+
+# Syncing the mailcow base directory
+echo -e "\033[1mSynchronizing mailcow base directory...\033[0m"
+rsync --delete -aH -e "ssh -o StrictHostKeyChecking=no \
+  -i \"${REMOTE_SSH_KEY}\" \
+  -p ${REMOTE_SSH_PORT}" \
+  "${SCRIPT_DIR}/../" root@${REMOTE_SSH_HOST}:"${SCRIPT_DIR}/../"
+ec=$?
+if [ ${ec} -ne 0 ] && [ ${ec} -ne 24 ]; then
+  >&2 echo -e "\e[31m[ERR]\e[0m - Could not transfer mailcow base directory to remote"
+  exit 1
+fi
+
+# Trigger a Redis save for a consistent Redis copy
+echo -ne "\033[1mRunning redis-cli save... \033[0m"
+docker exec $(docker ps -qf name=redis-mailcow) redis-cli save
+
+# Syncing volumes related to compose project
+# Same here: make sure destination exists
+for vol in $(docker volume ls -qf name="${CMPS_PRJ}"); do
+
+  mountpoint="$(docker inspect ${vol} | grep Mountpoint | cut -d '"' -f4)"
+
+  echo -e "\033[1mCreating remote mountpoint ${mountpoint} for ${vol}...\033[0m"
+
+  ssh -o StrictHostKeyChecking=no \
+    -i "${REMOTE_SSH_KEY}" \
+    ${REMOTE_SSH_HOST} \
+    -p ${REMOTE_SSH_PORT} \
+    mkdir -p "${mountpoint}"
+
+  if [[ "${vol}" =~ "mysql-vol-1" ]]; then
+
+    # Make sure a previous backup does not exist
+    rm -rf "${SCRIPT_DIR}/../_tmp_mariabackup/"
+
+    echo -e "\033[1mCreating consistent backup of MariaDB volume...\033[0m"
+    if ! docker run --rm \
+      --network $(docker network ls -qf name=${CMPS_PRJ}_) \
+      -v $(docker volume ls -qf name=${CMPS_PRJ}_mysql-vol-1):/var/lib/mysql/:ro \
+      --entrypoint= \
+      -v "${SCRIPT_DIR}/../_tmp_mariabackup":/backup \
+      ${SQLIMAGE} mariabackup --host mysql --user root --password ${DBROOT} --backup --target-dir=/backup 2>/dev/null ; then
+        >&2 echo -e "\e[31m[ERR]\e[0m - Could not create MariaDB backup on source"
+        rm -rf "${SCRIPT_DIR}/../_tmp_mariabackup/"
+        exit 1
+    fi
+
+    if ! docker run --rm \
+      --network $(docker network ls -qf name=${CMPS_PRJ}_) \
+      --entrypoint= \
+      -v "${SCRIPT_DIR}/../_tmp_mariabackup":/backup \
+      ${SQLIMAGE} mariabackup --prepare --target-dir=/backup 2> /dev/null ; then
+        >&2 echo -e "\e[31m[ERR]\e[0m - Could not transfer MariaDB backup to remote"
+        rm -rf "${SCRIPT_DIR}/../_tmp_mariabackup/"
+        exit 1
+    fi
+
+    chown -R 999:999 "${SCRIPT_DIR}/../_tmp_mariabackup"
+
+    echo -e "\033[1mSynchronizing MariaDB backup...\033[0m"
+    rsync --delete --info=progress2 -aH -e "ssh -o StrictHostKeyChecking=no \
+      -i \"${REMOTE_SSH_KEY}\" \
+      -p ${REMOTE_SSH_PORT}" \
+      "${SCRIPT_DIR}/../_tmp_mariabackup/" root@${REMOTE_SSH_HOST}:"${mountpoint}"
+    ec=$?
+    if [ ${ec} -ne 0 ] && [ ${ec} -ne 24 ]; then
+      >&2 echo -e "\e[31m[ERR]\e[0m - Could not transfer MariaDB backup to remote"
+      exit 1
+    fi
+
+    # Cleanup
+    rm -rf "${SCRIPT_DIR}/../_tmp_mariabackup/"
+
+  else
+
+    echo -e "\033[1mSynchronizing ${vol} from local ${mountpoint}...\033[0m"
+    rsync --delete --info=progress2 -aH -e "ssh -o StrictHostKeyChecking=no \
+      -i \"${REMOTE_SSH_KEY}\" \
+      -p ${REMOTE_SSH_PORT}" \
+      "${mountpoint}/" root@${REMOTE_SSH_HOST}:"${mountpoint}"
+    ec=$?
+    if [ ${ec} -ne 0 ] && [ ${ec} -ne 24 ]; then
+      >&2 echo -e "\e[31m[ERR]\e[0m - Could not transfer ${vol} from local ${mountpoint} to remote"
+      exit 1
+    fi
+  fi
+
+  echo -e "\e[32mCompleted\e[0m"
+
+done
+
+# Restart Dockerd on destination
+echo -ne "\033[1mRestarting Docker daemon on remote to detect new volumes... \033[0m"
+if ! ssh -o StrictHostKeyChecking=no \
+  -i "${REMOTE_SSH_KEY}" \
+  ${REMOTE_SSH_HOST} \
+  -p ${REMOTE_SSH_PORT} \
+  systemctl restart docker ; then
+    >&2 echo -e "\e[31m[ERR]\e[0m - Could not restart Docker daemon on remote"
+    exit 1
+fi
+echo "OK"
+
+echo -e "\033[1mPulling images on remote...\033[0m"
+if ! ssh -o StrictHostKeyChecking=no \
+  -i "${REMOTE_SSH_KEY}" \
+  ${REMOTE_SSH_HOST} \
+  -p ${REMOTE_SSH_PORT} \
+  docker-compose -f "${SCRIPT_DIR}/../docker-compose.yml" pull --no-parallel 2>&1 ; then
+    >&2 echo -e "\e[31m[ERR]\e[0m - Could not pull images on remote"
+fi
+
+echo -e "\033[1mForcing garbage cleanup on remote...\033[0m"
+if ! ssh -o StrictHostKeyChecking=no \
+  -i "${REMOTE_SSH_KEY}" \
+  ${REMOTE_SSH_HOST} \
+  -p ${REMOTE_SSH_PORT} \
+  ${SCRIPT_DIR}/../update.sh -f --gc ; then
+    >&2 echo -e "\e[31m[ERR]\e[0m - Could not cleanup old images on remote"
+fi
+
+echo -e "\e[32mDone\e[0m"
diff --git a/mailcow/src/mailcow-dockerized/helper-scripts/add-new-lang-keys.php b/mailcow/src/mailcow-dockerized/helper-scripts/add-new-lang-keys.php
new file mode 100644
index 0000000..19010f8
--- /dev/null
+++ b/mailcow/src/mailcow-dockerized/helper-scripts/add-new-lang-keys.php
@@ -0,0 +1,63 @@
+<?php
+
+function array_diff_key_recursive (array $arr1, array $arr2) {
+  $diff = array_diff_key($arr1, $arr2);
+  $intersect = array_intersect_key($arr1, $arr2);
+
+  foreach ($intersect as $k => $v) {
+    if (is_array($arr1[$k]) && is_array($arr2[$k])) {
+      $d = array_diff_key_recursive($arr1[$k], $arr2[$k]);
+
+      if ($d) {
+        $diff[$k] = $d;
+      }
+    }
+  }
+
+  return $diff;
+}
+
+// target lang
+$targetLang = $argv[1];
+
+if(empty($targetLang)) {
+  die('Please specify target lang as the first argument, to which you want to add missing keys from master lang (EN). Use the lowercase name,
+  for example `sk` for the Slovak language'."\n");
+}
+
+// load master lang
+$masterLang = file_get_contents(__DIR__.'/../data/web/lang/lang.en.json');
+$masterLang = json_decode($masterLang, true);
+
+// load target lang
+$lang = file_get_contents(__DIR__.'/../data/web/lang/lang.'.$targetLang.'.json');
+$lang = json_decode($lang, true);
+
+// compare lang keys
+$result = array_diff_key_recursive($masterLang, $lang);
+
+if(empty($result)) {
+  die('No new keys were added. Looks like target lang is up to date.'."\n");
+}
+
+foreach($result as $key => $val) {
+  // check if section key exists in target lang
+  if(array_key_exists($key, $lang)) {
+    // add only missing section keys
+    foreach ($val as $k => $v) {
+      $lang[$key][$k] = $v;
+    }
+    // sort keys
+    ksort($lang[$key]);
+  } else {
+    // add whole section
+    $lang[$key] = $val;
+    ksort($lang);
+  }
+}
+
+$lang = json_encode($lang, JSON_PRETTY_PRINT|JSON_UNESCAPED_UNICODE|JSON_UNESCAPED_SLASHES);
+file_put_contents(__DIR__.'/../data/web/lang/lang.'.$targetLang.'.json', $lang);
+
+echo 'Following new lang keys were added and need translation:'."\n";
+print_r($result);