#!/usr/bin/env bash # Support the Coolify-style one-line command while keeping a local copy for # later --cutover, --undo, and --attach commands. if [[ -z ${BASH_SOURCE[0]:-} || ! -f ${BASH_SOURCE[0]:-} ]]; then _cm_url="${COOLIFY_MIGRATE_URL:-https://coolify-migrate.grtsnx.com}" _cm_dst="${COOLIFY_MIGRATE_SCRIPT:-$HOME/coolify-migrate.sh}" _cm_tmp="${_cm_dst}.tmp.$$" mkdir -p "$(dirname "$_cm_dst")" || exit 1 curl -fsSL "$_cm_url" -o "$_cm_tmp" || { rm -f "$_cm_tmp"; exit 1; } /bin/bash -n "$_cm_tmp" || { rm -f "$_cm_tmp"; exit 1; } chmod 700 "$_cm_tmp" && mv -f "$_cm_tmp" "$_cm_dst" || { rm -f "$_cm_tmp"; exit 1; } # Let the first curl finish cleanly before replacing this stdin-driven shell. cat >/dev/null exec /bin/bash "$_cm_dst" "$@" fi # ============================================================================= # coolify-migrate — copy Coolify (or selected apps / databases / services) to # another server while the old one keeps running, then cut over when ready. # # * whole instance onto an empty server (users, settings, secrets, certs, data) # * selected resources onto an empty server or INTO an existing Coolify # * databases copied consistently while live (frozen for seconds, not stopped) # * every change on the new server is journaled: failures roll back automatically # # Run it ON the old server (e.g. from Coolify's own web terminal) or from any # machine with SSH access to both. Works with SSH keys, passwords, ssh-agent, # root or sudo (with or without password). # # curl -fsSL https://coolify-migrate.grtsnx.com | bash # # Verified download alternative: # curl -fsSL https://coolify-migrate.grtsnx.com -o coolify-migrate.sh # curl -fsSL https://gist.githubusercontent.com/grtsnx/e73980ff9ecc011e7ea843863ebe1cac/raw/coolify-migrate.sh.sha256 -o coolify-migrate.sh.sha256 # sha256sum -c coolify-migrate.sh.sha256 && bash coolify-migrate.sh # ./coolify-migrate.sh --src root@1.2.3.4 --dst root@5.6.7.8 --dst-key ~/.ssh/id_ed25519 # # Compatible with bash 3.2+ (macOS default) on the operator side. # ============================================================================= set -o pipefail TOOL_VERSION="2.5.0" TOOL_URL="https://coolify-migrate.grtsnx.com" SCRIPT_PATH=${BASH_SOURCE[0]} [[ $SCRIPT_PATH == /* ]] || SCRIPT_PATH="$PWD/$SCRIPT_PATH" # ---------------------------------------------------------------- defaults --- STATE_DIR="${COOLIFY_MIGRATE_HOME:-$HOME/.coolify-migrate}" RUN_ID="$(date +%Y%m%d-%H%M%S)" LOG="$STATE_DIR/logs/$RUN_ID.log" RWD="/var/lib/coolify-migrate" # work dir on both servers INSTALL_URL="${COOLIFY_INSTALL_URL:-https://cdn.coollabs.io/coolify/install.sh}" INSTALL_SHA256="${COOLIFY_INSTALL_SHA256:-}" # Key comments must never contain "coolify": Coolify's installer runs sed -i "/coolify/d" ~/.ssh/authorized_keys KEY_MARKER="cmig-ephemeral" SETUP_MARKER="cmig-setup" SUDOERS_FILE="/etc/sudoers.d/zz-coolify-migrate" PWCONF="/etc/ssh/sshd_config.d/00-coolify-migrate.conf" SETUP_KEY="$HOME/.ssh/coolify_migrate_ed25519" LIVE=0 INTERACTIVE=1 ASSUME_YES=0 PLAN_ONLY=0 FORCE_RELAY=0 ACCEPT_NEW_HOST_KEY=0 CONSOLIDATE=0 OPT_VOLUMES=1 OPT_BINDS=1 OPT_IMAGES=1 OPT_START=1 OPT_PAUSE_TASKS=1 OPT_IPREWRITE=1 OPT_COMPRESS=1 OPT_REMOTES=0 OPT_BRIDGE=1 CROSS_ARCH=0 AUTO_ROLLBACK=1 CUTOVER=0 UNDO=0 CLEANUP_ORPHANS=0 WHAT="" ENGINE="" TARGET_KIND="" SEL_UUIDS="" PHASE="" EXECUTING=0 OVERWRITE=0 SRC_LOCAL=0 DETACH=auto ATTACH=0 HANDED_OFF=0 WORKER_RC_FILE="" SUDO_GRANT_SRC=0 SUDO_GRANT_DST=0 PWGUIDE_SRC=0 PWGUIDE_DST=0 COOLIFY_VERSION_OVERRIDE="" DST_REACH="" SRC_HOST="" SRC_USER="" SRC_PORT="" SRC_AUTH="" SRC_KEY="" SRC_PASS="" SRC_SUDO="" DST_HOST="" DST_USER="" DST_PORT="" DST_AUTH="" DST_KEY="" DST_PASS="" DST_SUDO="" SRC_HOST_KEY="" DST_HOST_KEY="" FREEZE_TIMEOUT=30 TRANSFER_JOBS=2 KEY_TTL_HOURS=${COOLIFY_MIGRATE_KEY_TTL_HOURS:-24} TARGET_PROJECT_UUID="" # discovered facts (all validated on the way in) S_VERSION="" S_ARCH="" S_OS="" S_IP="" S_DATA=0 S_DBU=coolify S_DBN=coolify S_MID="" S_LOCAL_IP="" S_LOCAL_USER=root S_LOCAL_PUB="" S_COUNTS="" S_DOCKERCFG="" N_REMOTE=0 D_ARCH="" D_OS="" D_IP="" D_FREE=0 D_INODES=0 D_CPU=0 D_MEM_TOTAL=0 D_COOLIFY="" D_USERS=0 D_DATA="" D_MID="" D_HASDOCKER=0 D_JOURNAL=0 VOLS=() BINDS=() SKIPPED_BINDS=() IMGS=() RUNNING=() FQDNS=() DMOUNTS=() RES=() DPROJECTS=() PROJECT_MAPS=() # Workloads discovered on servers managed by the source Coolify. Records keep # their source server so data is always read from the host that owns it. MSERVERS=() MRES=() MVOLS=() MBINDS=() MIMGS=() MRUNNING=() MODE="direct" COMP="gzip -1 -c" DECOMP="gzip -dc" DSSH="" DT="" TRANSFER_REACH="" TRANSFER_PORT=22 # ---------------------------------------------------------------------- UI --- if [[ -t 1 && -z ${NO_COLOR:-} ]]; then B=$'\033[1m' D=$'\033[2m' R=$'\033[0m' RED=$'\033[31m' GRN=$'\033[32m' YEL=$'\033[33m' BLU=$'\033[34m' MAG=$'\033[35m' CYN=$'\033[36m' else B="" D="" R="" RED="" GRN="" YEL="" BLU="" MAG="" CYN="" fi TTY=/dev/tty cols() { local c; c=$(tput cols 2>/dev/null) || c=100; [[ $c -gt 20 ]] || c=100; echo "$c"; } # UTF-8 terminal? (override: COOLIFY_MIGRATE_ASCII=1/0). Non-UTF-8 locales get plain ASCII. case "${COOLIFY_MIGRATE_ASCII:-}" in 1) UTF8=0 ;; 0) UTF8=1 ;; *) case "${LC_ALL:-${LC_CTYPE:-${LANG:-}}}" in *[Uu][Tt][Ff]-8*|*[Uu][Tt][Ff]8*) UTF8=1 ;; *) UTF8=0 ;; esac ;; esac if ((UTF8)); then G_OK='✔' G_ERR='✖' G_WARN='▲' G_INFO='•' G_HR='─' G_SUB='›' G_PTR='❯' G_ON='●' G_OFF='○' G_ARR='→' G_DOT='·' G_ELL='…' G_DASH='—' G_NDASH='–' G_APPROX='≈' G_GE='≥' G_UP='↑' G_DOWN='↓' G_PARTY='🎉' SPIN=(⠋ ⠙ ⠹ ⠸ ⠼ ⠴ ⠦ ⠧ ⠇ ⠏) G_BFULL='━' G_BEMPTY='─' G_PFULL='█' G_PEMPTY='░' else G_OK='+' G_ERR='x' G_WARN='!' G_INFO='*' G_HR='-' G_SUB='>' G_PTR='>' G_ON='[x]' G_OFF='[ ]' G_ARR='->' G_DOT='|' G_ELL='...' G_DASH='-' G_NDASH='-' G_APPROX='~' G_GE='>=' G_UP='up' G_DOWN='down' G_PARTY='**' SPIN=('|' '/' '-' '\' '|' '/' '-' '\' '|' '/') G_BFULL='=' G_BEMPTY='-' G_PFULL='#' G_PEMPTY='.' fi # Coolify's installer cracks jokes while you wait. So do we. JOKES=( "Moving servers is like moving house, except the boxes are containers and nobody offers to help." "Your data is flying first class today: zstd-compressed, extra legroom." "It's not slow, it's thorough." "rsync has been moving files since 1996. It has seen things." "Docker volumes don't get homesick. They just get mounted somewhere else." "It's not DNS. There's no way it's DNS. ... It was DNS." "The old server is taking the news surprisingly well." "Packing your containers. Bubble wrap not included." "Postgres is holding still for pg_dump. Don't worry, it's a professional." "The two hardest things in computing: cache invalidation, naming things, and off-by-one errors." "Zero downtime is a mindset. Low downtime is a script." "No servers were harmed in the making of this migration." "Grab a coffee. The bytes have it from here." "Teaching your apps their new address. They're quick learners." "Somewhere, a sysadmin is doing this by hand at 3am. Not you." "Copying your secrets. Don't worry, we're not telling anyone." "Fun fact: 'cloud' is just someone else's computer. Now it's a different someone else's computer." ) quip() { printf ' %s%s %s%s\n' "$D" "$([[ $UTF8 == 1 ]] && printf '💬' || printf '>')" "${JOKES[RANDOM % ${#JOKES[@]}]}" "$R"; } log() { printf '[%s] %s\n' "$(date +%H:%M:%S)" "$*" >>"$LOG"; } info() { printf ' %s'"${G_INFO}"'%s %s\n' "$BLU" "$R" "$*"; log "INFO $*"; } ok() { printf ' %s'"${G_OK}"'%s %s\n' "$GRN" "$R" "$*"; log "OK $*"; } warn() { printf ' %s'"${G_WARN}"'%s %s\n' "$YEL" "$R" "$*"; log "WARN $*"; } err() { printf ' %s'"${G_ERR}"'%s %s\n' "$RED" "$R" "$*" >&2; log "ERR $*"; } die() { err "$*"; printf '\n %sLog:%s %s\n\n' "$D" "$R" "$LOG" >&2; exit 1; } self_cmd() { printf 'bash %q' "$SCRIPT_PATH"; } hr() { local i n; n=$(cols); ((n > 90)) && n=90; printf '%s' "$D"; for ((i = 0; i < n; i++)); do printf ''"${G_HR}"''; done; printf '%s\n' "$R"; } STEP_N=0 STEP_T=0 RUN_START=$SECONDS # Bar of N cells, P percent filled bar() { # pct width local p=$1 w=$2 f i out="" ((p > 100)) && p=100; ((p < 0)) && p=0 f=$((p * w / 100)) for ((i = 0; i < w; i++)); do if ((i < f)); then out+="$G_BFULL"; else out+="$G_BEMPTY"; fi; done printf '%s' "$out" } section() { STEP_N=$((STEP_N + 1)) ((STEP_N > STEP_T)) && STEP_T=$STEP_N local pct=$(((STEP_N - 1) * 100 / STEP_T)) printf '\n%s%s[%d/%d]%s %s%s%s %s%s%s %s%d%% %s %s%s\n' "$B" "$MAG" "$STEP_N" "$STEP_T" "$R" "$B" "$1" "$R" \ "$CYN" "$(bar "$pct" 16)" "$R" "$D" "$pct" "$G_DOT" "$(fmt_dur $((SECONDS - RUN_START)))" "$R" log "=== [$STEP_N/$STEP_T] $1" } banner() { local w; w=$(cols) echo if ((UTF8 && w >= 58)); then printf '%s%s' "$B" "$CYN" cat <<'EOF' ██████╗ ██████╗ ██████╗ ██╗ ██╗███████╗██╗ ██╗ ██╔════╝██╔═══██╗██╔═══██╗██║ ██║██╔════╝╚██╗ ██╔╝ ██║ ██║ ██║██║ ██║██║ ██║█████╗ ╚████╔╝ ██║ ██║ ██║██║ ██║██║ ██║██╔══╝ ╚██╔╝ ╚██████╗╚██████╔╝╚██████╔╝███████╗██║██║ ██║ ╚═════╝ ╚═════╝ ╚═════╝ ╚══════╝╚═╝╚═╝ ╚═╝ EOF printf '%s' "$MAG" cat <<'EOF' ███╗ ███╗██╗ ██████╗ ██████╗ █████╗ ████████╗███████╗ ████╗ ████║██║██╔════╝ ██╔══██╗██╔══██╗╚══██╔══╝██╔════╝ ██╔████╔██║██║██║ ███╗██████╔╝███████║ ██║ █████╗ ██║╚██╔╝██║██║██║ ██║██╔══██╗██╔══██║ ██║ ██╔══╝ ██║ ╚═╝ ██║██║╚██████╔╝██║ ██║██║ ██║ ██║ ███████╗ ╚═╝ ╚═╝╚═╝ ╚═════╝ ╚═╝ ╚═╝╚═╝ ╚═╝ ╚═╝ ╚══════╝ EOF elif ((UTF8)); then printf '%s%s' "$B" "$CYN" cat <<'EOF' ┌─┐┌─┐┌─┐┬ ┬┌─┐┬ ┬ ┌┬┐┬┌─┐┬─┐┌─┐┌┬┐┌─┐ │ │ ││ ││ │├┤ └┬┘───│││││ ┬├┬┘├─┤ │ ├┤ └─┘└─┘└─┘┴─┘┴└ ┴ ┴ ┴┴└─┘┴└─┴ ┴ ┴ └─┘ EOF else printf '%s%s' "$B" "$CYN" cat <<'EOF' _ _ __ _ _ __ ___ ___| (_)/ _|_ _ ___ _ __ (_)__ _ _ _ __ _| |_ ___ / _/ _ \/ _ \ | | _| || |___| ' \| / _` | '_/ _` | _/ -_) \__\___/\___/_|_|_| \_, | |_|_|_|_\__, |_| \__,_|\__\___| |__/ |___/ EOF fi printf '%s\n %sone-click Coolify server migration %s v%s%s\n' "$R" "$D" "$G_DOT" "$TOOL_VERSION" "$R" printf ' %sbuilt by %s%sgrtsnx%s %s %s%shttps://github.com/grtsnx%s\n\n' "$D" "$R" "$B" "$R" "$G_DOT" "$R" "$CYN" "$R" } hsize() { awk -v b="${1:-0}" 'BEGIN{split("B KB MB GB TB",u);i=1;while(b>=1024&&i<5){b/=1024;i++}printf (i==1?"%d %s":"%.1f %s"),b,u[i]}'; } fmt_dur() { local s=$1; if ((s >= 60)); then printf '%dm%02ds' $((s / 60)) $((s % 60)); else printf '%ds' "$s"; fi; } # Widget locals are __-prefixed so they never shadow the caller's result variable. ui_ask() { # var prompt [default] local __v=$1 __p=$2 __d=${3:-} __a="" if ((!INTERACTIVE)); then printf -v "$__v" '%s' "$__d"; return; fi printf ' %s?%s %s%s%s ' "$CYN" "$R" "$B" "$__p" "$R" >"$TTY" [[ -n $__d ]] && printf '%s(%s)%s ' "$D" "$__d" "$R" >"$TTY" IFS= read -r __a <"$TTY" printf -v "$__v" '%s' "${__a:-$__d}" } ui_secret() { # var prompt local __v=$1 __a="" printf ' %s?%s %s%s%s ' "$CYN" "$R" "$B" "$2" "$R" >"$TTY" IFS= read -rs __a <"$TTY"; printf '%s'"${G_INFO}"''"${G_INFO}"''"${G_INFO}"''"${G_INFO}"''"${G_INFO}"''"${G_INFO}"'%s\n' "$D" "$R" >"$TTY" printf -v "$__v" '%s' "$__a" } ui_confirm() { # prompt [y|n] -> 0 yes local __d=${2:-n} __a __hint="y/N" [[ $__d == y ]] && __hint="Y/n" if ((!INTERACTIVE)); then [[ $__d == y ]]; return; fi printf ' %s?%s %s%s%s %s[%s]%s ' "$CYN" "$R" "$B" "$1" "$R" "$D" "$__hint" "$R" >"$TTY" IFS= read -r __a <"$TTY"; __a=${__a:-$__d} [[ $__a == [yY]* ]] } # Read one keypress: prints up|down|space|enter| ui_key() { local __k __k2 IFS= read -rsn1 __k <"$TTY" case $__k in $'\x1b') IFS= read -rsn2 __k2 <"$TTY" case $__k2 in '[A') echo up ;; '[B') echo down ;; *) echo esc ;; esac ;; '') echo enter ;; ' ') echo space ;; *) printf '%s\n' "$__k" ;; esac } # Arrow-key single choice. Result: index in var. ui_select() { # var title opts... local __v=$1 __title=$2; shift 2 local __opts=("$@") __n=$# __cur=0 __i __k if ((!INTERACTIVE)); then printf -v "$__v" '%s' 0; return; fi printf ' %s?%s %s%s%s %s('"${G_UP}"'/'"${G_DOWN}"', enter)%s\n' "$CYN" "$R" "$B" "$__title" "$R" "$D" "$R" >"$TTY" tput civis 2>/dev/null while :; do for ((__i = 0; __i < __n; __i++)); do if ((__i == __cur)); then printf ' %s'"${G_PTR}"' %s%s\033[K\n' "$CYN" "${__opts[__i]}" "$R" >"$TTY" else printf ' %s%s%s\033[K\n' "$D" "${__opts[__i]}" "$R" >"$TTY"; fi done __k=$(ui_key) case $__k in up|k) __cur=$(((__cur - 1 + __n) % __n)) ;; down|j) __cur=$(((__cur + 1) % __n)) ;; [1-9]) ((__k <= __n)) && __cur=$((__k - 1)) ;; enter) break ;; esac printf '\033[%dA' "$__n" >"$TTY" done tput cnorm 2>/dev/null printf -v "$__v" '%s' "$__cur" } # Arrow-key checklist. defaults: "1 0 1 ..." ; result var: "1 0 1 ..." ui_multi() { # var title defaults opts... local __v=$1 __title=$2 __defs=$3; shift 3 local __opts=("$@") __n=$# __cur=0 __i __k __box __ptr __any0 __sel=() read -r -a __sel <<<"$__defs" if ((INTERACTIVE)); then printf ' %s?%s %s%s%s %s('"${G_UP}"'/'"${G_DOWN}"' move, space toggle, a all, enter confirm)%s\n' "$CYN" "$R" "$B" "$__title" "$R" "$D" "$R" >"$TTY" tput civis 2>/dev/null while :; do for ((__i = 0; __i < __n; __i++)); do __box="${D}${G_OFF}${R}" __ptr=" " [[ ${__sel[__i]:-0} == 1 ]] && __box="${GRN}${G_ON}${R}" ((__i == __cur)) && __ptr="${CYN}${G_PTR}${R}" printf ' %s %s %s\033[K\n' "$__ptr" "$__box" "${__opts[__i]}" >"$TTY" done __k=$(ui_key) case $__k in up|k) __cur=$(((__cur - 1 + __n) % __n)) ;; down|j) __cur=$(((__cur + 1) % __n)) ;; space) if [[ ${__sel[__cur]:-0} == 1 ]]; then __sel[__cur]=0; else __sel[__cur]=1; fi ;; a) __any0=0; for ((__i = 0; __i < __n; __i++)); do [[ ${__sel[__i]:-0} == 0 ]] && __any0=1; done for ((__i = 0; __i < __n; __i++)); do __sel[__i]=$__any0; done ;; enter) break ;; esac printf '\033[%dA' "$__n" >"$TTY" done tput cnorm 2>/dev/null fi for ((__i = 0; __i < __n; __i++)); do __sel[__i]=${__sel[__i]:-0}; done printf -v "$__v" '%s' "${__sel[*]}" } # Run a command in the background with spinner + live tail of its output. ui_run() { # title cmd... local title=$1; shift local start=$SECONDS i=0 last w rc log "RUN $title :: $*" ("$@") >>"$LOG" 2>&1 /dev/null w=$(($(cols) - ${#title} - 20)); ((w < 10)) && w=10 while kill -0 "$pid" 2>/dev/null; do last=$(tail -n 1 "$LOG" 2>/dev/null | sed $'s/\033\\[[0-9;?]*[A-Za-z]//g' | tr -d '\r\033' | cut -c1-"$w") printf '\r\033[K %s%s%s %s %s%s '"${G_DOT}"' %s%s' "$CYN" "${SPIN[i % 10]}" "$R" "$title" "$D" "$(fmt_dur $((SECONDS - start)))" "$last" "$R" i=$((i + 1)); sleep "$tick" done tput cnorm 2>/dev/null; printf '\r\033[K' fi wait "$pid"; rc=$? if ((rc == 0)); then ok "$title ${D}($(fmt_dur $((SECONDS - start))))${R}" else err "$title failed (exit $rc)"; tail -n 12 "$LOG" | tr -d '\000-\010\013-\037\177' | sed 's/^/ /' >&2; fi return $rc } # Run a slow command with the same live spinner as ui_run, but return its # stdout to the caller. This keeps discovery/import steps interactive without # losing the structured output they need to parse. ui_capture() { # result-var title cmd... local __v=$1 title=$2; shift 2 local start=$SECONDS i=0 last w rc tmp="$CTL_DIR/capture.$$.${RANDOM:-0}" mkdir -p "$CTL_DIR" "$(dirname "$LOG")"; chmod 700 "$CTL_DIR" "$(dirname "$LOG")" 2>/dev/null log "RUN $title :: $*" ( umask 077; : >"$tmp" ) ("$@") >"$tmp" 2>>"$LOG" /dev/null w=$(($(cols) - ${#title} - 20)); ((w < 10)) && w=10 while kill -0 "$pid" 2>/dev/null; do last=$(tail -n 1 "$LOG" 2>/dev/null | sed $'s/\033\\[[0-9;?]*[A-Za-z]//g' | tr -d '\r\033' | cut -c1-"$w") printf '\r\033[K %s%s%s %s %s%s '"${G_DOT}"' %s%s' "$CYN" "${SPIN[i % 10]}" "$R" "$title" "$D" "$(fmt_dur $((SECONDS - start)))" "$last" "$R" i=$((i + 1)); sleep "$tick" done tput cnorm 2>/dev/null; printf '\r\033[K' fi wait "$pid"; rc=$? printf -v "$__v" '%s' "$(cat "$tmp")" rm -f "$tmp" if ((rc == 0)); then ok "$title ${D}($(fmt_dur $((SECONDS - start))))${R}" else err "$title failed (exit $rc)"; tail -n 12 "$LOG" | tr -d '\000-\010\013-\037\177' | sed 's/^/ /' >&2; fi return $rc } # rsync --info=progress2 line -> "▕████░░░░▏ 63% · 580 MB · 87.9 MB/s · ETA 0:12" progress_line() { local bytes pct rate eta rest read -r bytes pct rate eta rest <<<"$1" pct=${pct%\%}; [[ $pct =~ ^[0-9]+$ ]] || return 0 bytes=${bytes//,/}; [[ $bytes =~ ^[0-9]+$ ]] || bytes=0 [[ $eta == *:* ]] || eta="--" local b="" i f=$((pct * 24 / 100)) for ((i = 0; i < 24; i++)); do if ((i < f)); then b+="$G_PFULL"; else b+="$G_PEMPTY"; fi; done printf '\r\033[K %s%s%s %3d%% %s %s %s %s %s ETA %s%s' "$CYN" "$b" "$R" "$pct" "$G_DOT" "$(hsize "$bytes")" "$G_DOT" "${rate:-?}" "$G_DOT" "$eta" "$D$R" } # Run a command that prints rsync-style progress (\r separated); render in place. # Lines "@@ text" become sub-headers. ui_stream() { # title cmd... local title=$1; shift local start=$SECONDS chunk l w rc w=$(($(cols) - 8)) info "$title" log "RUN $title :: $*" rm -f "$CTL_DIR/rc" { ("$@") 2>&1 >"$LOG" case $l in '@@RC '*) echo "${l#@@RC }" >"$CTL_DIR/rc" ;; '@@ '*) printf '\r\033[K %s'"${G_SUB}"'%s %s\n' "$MAG" "$R" "$(clean "${l#@@ }")" ;; *'%'*'/s'*|*'%'*xfr*) progress_line "$l" ;; *rsync:*|*error*|*ERROR*|*No\ such*) printf '\r\033[K %s%s%s\n' "$YEL" "$(clean "$l" | cut -c1-"$w")" "$R" ;; esac done <<<"$chunk" done printf '\r\033[K' rc=$(cat "$CTL_DIR/rc" 2>/dev/null || echo 1) if [[ $rc == 0 ]]; then ok "$title ${D}($(fmt_dur $((SECONDS - start))))${R}" else err "$title failed (exit $rc)"; fi return "$rc" } # -------------------------------------------------------------- validation --- # Everything read from a server is untrusted. Validate before it reaches # arithmetic, a shell, SQL, rsync or the terminal. num() { [[ $1 =~ ^[0-9]{1,18}$ ]] && printf '%s' "$1" || printf 0; } is_ip4() { [[ $1 =~ ^([0-9]{1,3})\.([0-9]{1,3})\.([0-9]{1,3})\.([0-9]{1,3})$ ]] && ((BASH_REMATCH[1] < 256 && BASH_REMATCH[2] < 256 && BASH_REMATCH[3] < 256 && BASH_REMATCH[4] < 256)); } is_private_ip() { [[ $1 =~ ^(10\.|192\.168\.|172\.(1[6-9]|2[0-9]|3[01])\.|127\.|100\.(6[4-9]|[7-9][0-9]|1[01][0-9]|12[0-7])\.) ]]; } is_name() { [[ $1 =~ ^[A-Za-z0-9][A-Za-z0-9_.-]{0,250}$ ]]; } is_uuid() { [[ $1 =~ ^[A-Za-z0-9_-]{1,64}$ ]]; } is_user() { [[ $1 =~ ^[a-z_][a-z0-9_-]{0,31}$ ]]; } is_port() { [[ $1 =~ ^[0-9]{1,5}$ ]] && (($1 > 0 && $1 < 65536)); } is_fqdn() { [[ $1 =~ ^[A-Za-z0-9*]([A-Za-z0-9.-]{0,251}[A-Za-z0-9])?$ ]]; } is_image() { [[ $1 =~ ^[a-z0-9][a-z0-9._/:@-]{0,254}$ ]]; } is_version() { [[ $1 =~ ^[0-9A-Za-z.+-]{1,40}$ ]]; } is_abspath() { [[ $1 =~ ^/[A-Za-z0-9._@+/-]+$ && $1 != *..* ]]; } clean() { printf '%s' "$1" | tr -d '\000-\010\013-\037\177'; } # strip control chars before printing # Bind mounts are only copied from data locations, never system paths. bind_ok() { local p=${1%/} is_abspath "$p" || return 1 case $p in /etc|/etc/*|/usr|/usr/*|/var|/var/lib|/var/lib/docker|/var/lib/docker/*|/var/log|/var/log/*|/var/run|/var/run/*) return 1 ;; /root|/root/.ssh|/root/.ssh/*|/home|/home/*/.ssh|/home/*/.ssh/*|/boot|/boot/*|/opt|/srv|/mnt|/media|/data) return 1 ;; /tmp|/tmp/*|/run|/run/*|/snap|/snap/*|/proc|/proc/*|/sys|/sys/*|/dev|/dev/*) return 1 ;; /bin|/bin/*|/sbin|/sbin/*|/lib|/lib/*|/lib32|/lib32/*|/lib64|/lib64/*|/libx32|/libx32/*) return 1 ;; /data/coolify|/data/coolify/*|"$RWD"|"$RWD"/*) return 1 ;; esac [[ $p == /home/* && $p != /home/*/* ]] && return 1 # a whole home directory [[ $p == /*/* ]] # at least two levels deep } # ------------------------------------------------------------------- utils --- hv() { local _n="${1}_${2}"; printf '%s' "${!_n}"; } setv() { printf -v "${1}_${2}" '%s' "$3"; } sq() { local s=${1//\'/\'\\\'\'}; printf "'%s'" "$s"; } # POSIX single-quote def() { IFS= read -r -d '' "$1" || true; } # heredoc -> var target() { printf '%s@%s' "$(hv "$1" USER)" "$(hv "$1" HOST)"; } is_local() { [[ $1 == SRC && $SRC_LOCAL == 1 ]]; } src_cmd() { if ((SRC_LOCAL)); then printf '%s' "$1"; else printf 'ssh %s "%s"' "$(target SRC)" "$1"; fi; } US=$'\x1f' # field separator for records that may contain spaces # --------------------------------------------------------------------- SSH --- CTL_DIR="$(mktemp -d "${TMPDIR:-/tmp}/cm.XXXXXX")" || exit 1 CTL_DIR="${CTL_DIR%/}" # Unix socket paths are short-limited (~104 chars on macOS) if ((${#CTL_DIR} > 60)); then rmdir "$CTL_DIR"; CTL_DIR="$(mktemp -d /tmp/cm.XXXXXX)" || exit 1; fi chmod 700 "$CTL_DIR" ASKPASS="$CTL_DIR/askpass" printf '#!/bin/sh\nprintf "%%s\\n" "$CM_PW"\n' >"$ASKPASS"; chmod 700 "$ASKPASS" ssh_minor() { ssh -V 2>&1 | sed -n 's/^OpenSSH_\([0-9]*\)\.\([0-9]*\).*/\1 \2/p'; } password_supported() { command -v sshpass >/dev/null 2>&1 && return 0 local v; v=$(ssh_minor); set -- $v [[ -n ${1:-} ]] && { (($1 > 8)) || (($1 == 8 && ${2:-0} >= 4)); } } # First contact with a host: show its fingerprint and ask before trusting it (TOFU with consent). approve_new_host_key() { # role fingerprints local r=$1 fingerprints=$2 expected expected=$(hv "$r" HOST_KEY) if [[ -n $expected ]]; then grep -Fq -- "$expected " <<<"$fingerprints" && return 0 printf 'Pinned host key %s does not match the key offered by the server.\n' "$expected" >&2 return 1 fi ((INTERACTIVE)) && { ui_confirm "Trust this server?" y; return; } ((ACCEPT_NEW_HOST_KEY)) && return 0 printf 'Refusing an unpinned host key in unattended mode. Use --%s-host-key SHA256:... (recommended) or --accept-new-host-key.\n' \ "$(tr A-Z a-z <<<"$r")" >&2 return 1 } confirm_host_key() { # role local r=$1 host port kh="$HOME/.ssh/known_hosts" spec scanned fp host=$(hv "$r" HOST); port=$(hv "$r" PORT) spec=$host; [[ $port != 22 ]] && spec="[$host]:$port" mkdir -p "$HOME/.ssh" && chmod 700 "$HOME/.ssh" if ssh-keygen -F "$spec" -f "$kh" >/dev/null 2>&1; then if [[ -n $(hv "$r" HOST_KEY) ]]; then fp=$(ssh-keygen -F "$spec" -f "$kh" 2>/dev/null | ssh-keygen -lf - 2>/dev/null | awk '{print $2" ("$NF")"}') approve_new_host_key "$r" "$fp" || die "Stored host key does not match the pinned fingerprint." fi return 0 fi scanned="$CTL_DIR/scan-$r" ssh-keyscan -T 10 -p "$port" -t ed25519,ecdsa,rsa "$host" 2>/dev/null >"$scanned" [[ -s $scanned ]] || return 0 # unreachable: the connect step will explain fp=$(ssh-keygen -lf "$scanned" 2>/dev/null | awk '{print $2" ("$NF")"}' | head -n 3) if ((INTERACTIVE)); then info "First connection to ${B}$host${R}. Its SSH host key fingerprint is:" printf '%s\n' "$fp" | sed "s/^/ ${GRN}/;s/\$/${R}/" info "${D}(Your cloud console usually shows this in the server's boot log — compare if you want to be sure.)${R}" fi approve_new_host_key "$r" "$fp" || die "Host key not trusted — aborted." ((INTERACTIVE)) || log "Pinned/explicit trust accepted for new host key $spec" cat "$scanned" >>"$kh"; chmod 600 "$kh" } open_master() { # role local r=$1 t; t=$(target "$r") local -a o=(-M -S "$CTL_DIR/$r.sock" -o ControlPersist=4h -o StrictHostKeyChecking=yes -o ServerAliveInterval=15 -o ServerAliveCountMax=4 -o ConnectTimeout=15 -p "$(hv "$r" PORT)" -f -N) case $(hv "$r" AUTH) in key) o+=(-i "$(hv "$r" KEY)" -o IdentitiesOnly=yes -o PasswordAuthentication=no -o KbdInteractiveAuthentication=no) ssh "${o[@]}" -- "$t" ;; password) o+=(-o PreferredAuthentications=password,keyboard-interactive -o PubkeyAuthentication=no -o NumberOfPasswordPrompts=1) if command -v sshpass >/dev/null 2>&1; then SSHPASS="$(hv "$r" PASS)" sshpass -e ssh "${o[@]}" -- "$t" else CM_PW="$(hv "$r" PASS)" SSH_ASKPASS="$ASKPASS" SSH_ASKPASS_REQUIRE=force DISPLAY="${DISPLAY:-:0}" ssh "${o[@]}" -- "$t" "$2"; } close_master() { is_local "$1" && return 0; [[ -S "$CTL_DIR/$1.sock" ]] && ssh -S "$CTL_DIR/$1.sock" -O exit -- "$(target "$1")" >/dev/null 2>&1; } rssh() { # role args... (raw ssh over the multiplexed connection; local shell for a local source) local r=$1; shift if is_local "$r"; then sh -c "$*"; return; fi [[ -S "$CTL_DIR/$r.sock" ]] || return 255 ssh -S "$CTL_DIR/$r.sock" -o ControlMaster=no -o LogLevel=ERROR -p "$(hv "$r" PORT)" -- "$(target "$r")" "$@" } # Run a bash script (string) on a role as root. Extra args: KEY=VALUE, exported (safely quoted) first. rscript() { # role script [K=V...] local r=$1 body=$2 kv pre; shift 2 pre="export LC_ALL=C RWD=$(sq "$RWD") MARKER_EPH=$(sq "$KEY_MARKER") MARKER_SETUP=$(sq "$SETUP_MARKER");"$'\n' for kv in "$@"; do pre+="export ${kv%%=*}=$(sq "${kv#*=}");"$'\n'; done printf '%s\n%s\n%s\n' "$pre" "$RS_PRELUDE" "$body" | rssh "$r" "$(hv "$r" SUDO) bash -s" } # ----------------------------------------------------------- remote scripts -- # Prepended to every remote script. Remote side is always bash on Linux. def RS_PRELUDE <<'EOF' set -o pipefail sq() { local s=${1//\'/\'\\\'\'}; printf "'%s'" "$s"; } oneline() { printf '%s' "$1" | tr -d '\000-\037\177' | head -c 1024; } emit() { printf '%s=%s\n' "$1" "$(oneline "$2")"; } ip4() { printf '%s' "$1" | grep -Eqx '([0-9]{1,3}\.){3}[0-9]{1,3}'; } pubip() { local ip u for u in https://ifconfig.io https://api.ipify.org https://ipv4.icanhazip.com; do ip=$(curl -4fsS --max-time 6 "$u" 2>/dev/null | head -c 64 | tr -d '\r\n ') ip4 "$ip" && { printf '%s' "$ip"; return; } done; } envval() { grep -E "^$1=" /data/coolify/source/.env 2>/dev/null | head -n 1 | cut -d= -f2-; } dbu() { local v; v=$(envval DB_USERNAME); printf '%s' "${v:-coolify}"; } dbn() { local v; v=$(envval DB_DATABASE); printf '%s' "${v:-coolify}"; } psqlc() { docker exec coolify-db psql -U "$(dbu)" -d "$(dbn)" -AtX -v ON_ERROR_STOP=1 "$@" >"$RWD/journal"; } homes() { { getent passwd 2>/dev/null || cat /etc/passwd; } | cut -d: -f6 | sort -u; } EOF def RS_DISCOVER_SRC <<'EOF' command -v docker >/dev/null || { emit FATAL "docker is not installed"; exit 0; } docker container inspect coolify >/dev/null 2>&1 || { emit FATAL "no 'coolify' container found - is Coolify v4 installed here?"; exit 0; } img=$(docker container inspect -f '{{.Config.Image}}' coolify); emit VERSION "${img##*:}" emit ARCH "$(uname -m)"; . /etc/os-release 2>/dev/null; emit OS "${PRETTY_NAME:-unknown}" emit MID "$(cat /etc/machine-id 2>/dev/null)" emit IP "$(pubip)" emit DATA "$(du -sb /data/coolify 2>/dev/null | cut -f1)" emit DBU "$(dbu)"; emit DBN "$(dbn)" q() { psqlc -F $'\x1f' -c "$1" 2>/dev/null; } IFS=$'\x1f' read -r lip luser kuuid <<<"$(q "select s.ip, s.\"user\", coalesce(pk.uuid,'') from servers s left join private_keys pk on pk.id=s.private_key_id where s.id=0")" emit LOCALIP "$lip"; emit LOCALUSER "${luser:-root}" [ -n "$kuuid" ] && [ -f "/data/coolify/ssh/keys/ssh_key@$kuuid" ] && emit LOCALPUB "$(ssh-keygen -y -f "/data/coolify/ssh/keys/ssh_key@$kuuid" 2>/dev/null)" emit NREMOTE "$(q "select count(*) from servers where id<>0")" lhome=$(getent passwd "${luser:-root}" 2>/dev/null | cut -d: -f6); lhome=${lhome:-/root} [ -f "$lhome/.docker/config.json" ] && emit DOCKERCFG "$lhome/.docker/config.json" c() { q "select count(*) from $1" || echo 0; } dbt="standalone_postgresqls standalone_mysqls standalone_mariadbs standalone_mongodbs standalone_redis standalone_keydbs standalone_dragonflies standalone_clickhouses" dbs=0; for t in $dbt; do n=$(c "$t"); dbs=$((dbs + ${n:-0})); done emit COUNTS "$(c applications) apps, $(c services) services, $dbs databases, $(c servers) servers" # resources on this server, for selective migration: # type US uuid US name US project US environment US project_uuid clean="translate(%s, E'\n\r\t\x1f', ' ')" sel() { printf "select '%s', x.uuid, $clean, $clean, $clean, p.uuid" "$1" x.name p.name e.name; } q "$(sel application) from applications x join environments e on e.id=x.environment_id join projects p on p.id=e.project_id join standalone_dockers d on d.id=x.destination_id and x.destination_type like '%StandaloneDocker' where d.server_id=0 and x.deleted_at is null" | sed 's/^/RES=/' q "$(sel service) from services x join environments e on e.id=x.environment_id join projects p on p.id=e.project_id where x.server_id=0 and x.deleted_at is null" | sed 's/^/RES=/' for t in $dbt; do q "$(sel database) from $t x join environments e on e.id=x.environment_id join projects p on p.id=e.project_id join standalone_dockers d on d.id=x.destination_id and x.destination_type like '%StandaloneDocker' where d.server_id=0 and x.deleted_at is null" | sed 's/^/RES=/' done # public domains served from this server (DNS checklist) q "select a.fqdn from applications a join standalone_dockers d on d.id=a.destination_id where a.destination_type like '%StandaloneDocker' and d.server_id=0 and a.fqdn is not null union all select sa.fqdn from service_applications sa join services s on s.id=sa.service_id where s.server_id=0 and sa.fqdn is not null union all select fqdn from instance_settings where fqdn is not null" | tr ',' '\n' | sed -n 's#^[a-z]*://\([^/:]*\).*#FQDN=\1#p' | sort -u # volumes: name|bytes|in-use|mountpoint|labels(base64)|containers docker volume ls -q | while read -r v; do case $v in coolify-db|coolify-redis|buildx_buildkit_*) continue ;; esac users=$(docker ps -a --filter volume="$v" --format '{{.Names}}' | tr '\n' ',' ); users=${users%,} [ -n "$users" ] && ! printf '%s' "$users" | tr ',' '\n' | grep -qv -e '^coolify' -e '^buildx_buildkit' && continue # Coolify/BuildKit internals mp=$(docker volume inspect -f '{{.Mountpoint}}' "$v") lb=$(docker volume inspect -f '{{range $k,$v := .Labels}}{{$k}}={{$v}}{{"\n"}}{{end}}' "$v" | base64 | tr -d '\n') printf 'VOL=%s|%s|%s|%s|%s|%s\n' "$(oneline "$v")" "$(du -sb "$mp" 2>/dev/null | cut -f1)" "${users:+1}" "$(oneline "$mp")" "$lb" "$(oneline "$users")" done ids=$(docker ps -aq --filter label=coolify.managed=true) if [ -n "$ids" ]; then for id in $ids; do n=$(docker inspect -f '{{.Name}}' "$id" | sed 's#^/##') docker inspect -f '{{range .Mounts}}{{if eq .Type "bind"}}{{.Source}}{{"\n"}}{{end}}{{end}}' "$id" | while read -r p; do [ -n "$p" ] && [ -e "$p" ] && printf 'BIND=%s|%s|%s\n' "$(oneline "$p")" "$(du -sb "$p" 2>/dev/null | cut -f1)" "$(oneline "$n")" done done docker inspect -f '{{.Image}}' $ids | sort -u >"$RWD/imgids" 2>/dev/null || { mkdir -p "$RWD" && chmod 700 "$RWD" && docker inspect -f '{{.Image}}' $ids | sort -u >"$RWD/imgids"; } docker images --no-trunc --format '{{.ID}} {{.Repository}}:{{.Tag}}' | grep -v '' | while read -r id ref; do grep -qx "$id" "$RWD/imgids" && printf 'IMG=%s|%s\n' "$(oneline "$ref")" "$(docker image inspect -f '{{.Size}}' "$ref")" done rm -f "$RWD/imgids" fi docker ps --format '{{.Names}}|{{.Image}}' | sed 's/^/RUN=/' true EOF # SOURCE CONTROL PLANE: inventory workloads that actually run on managed # servers. The source Coolify already owns the SSH keys used for these hosts. def RS_DISCOVER_MANAGED <<'EOF' q() { psqlc -F $'\x1f' -c "$1" 2>/dev/null; } clean="translate(%s, E'\\n\\r\\t\\x1f', ' ')" sel() { printf "select '%s', x.uuid, $clean, $clean, $clean, %s, s.name" "$1" x.name p.name e.name "$2"; } q "$(sel application d.server_id) from applications x join environments e on e.id=x.environment_id join projects p on p.id=e.project_id join standalone_dockers d on d.id=x.destination_id and x.destination_type like '%StandaloneDocker' join servers s on s.id=d.server_id where d.server_id<>0 and x.deleted_at is null" | while IFS=$'\x1f' read -r typ uuid name project env sid sname; do printf 'MRES=%s\x1f%s\x1f%s\x1f%s\x1f%s\x1f%s\x1f%s\n' "$typ" "$uuid" "$name" "$project" "$env" "$sid" "$sname" done q "$(sel application sd.server_id) from applications x join environments e on e.id=x.environment_id join projects p on p.id=e.project_id join swarm_dockers sd on sd.id=x.destination_id and x.destination_type like '%SwarmDocker' join servers s on s.id=sd.server_id where sd.server_id<>0 and x.deleted_at is null" | while IFS=$'\x1f' read -r typ uuid name project env sid sname; do printf 'MRES=%s\x1f%s\x1f%s\x1f%s\x1f%s\x1f%s\x1f%s\n' "$typ" "$uuid" "$name" "$project" "$env" "$sid" "$sname" done q "$(sel service x.server_id) from services x join environments e on e.id=x.environment_id join projects p on p.id=e.project_id join servers s on s.id=x.server_id where x.server_id<>0 and x.deleted_at is null" | while IFS=$'\x1f' read -r typ uuid name project env sid sname; do printf 'MRES=%s\x1f%s\x1f%s\x1f%s\x1f%s\x1f%s\x1f%s\n' "$typ" "$uuid" "$name" "$project" "$env" "$sid" "$sname" done for t in standalone_postgresqls standalone_mysqls standalone_mariadbs standalone_mongodbs standalone_redis standalone_keydbs standalone_dragonflies standalone_clickhouses; do q "$(sel database d.server_id) from $t x join environments e on e.id=x.environment_id join projects p on p.id=e.project_id join standalone_dockers d on d.id=x.destination_id and x.destination_type like '%StandaloneDocker' join servers s on s.id=d.server_id where d.server_id<>0 and x.deleted_at is null" | while IFS=$'\x1f' read -r typ uuid name project env sid sname; do printf 'MRES=%s\x1f%s\x1f%s\x1f%s\x1f%s\x1f%s\x1f%s\n' "$typ" "$uuid" "$name" "$project" "$env" "$sid" "$sname" done done q "select a.fqdn from applications a join standalone_dockers d on d.id=a.destination_id where a.destination_type like '%StandaloneDocker' and d.server_id<>0 and a.fqdn is not null union all select sa.fqdn from service_applications sa join services s on s.id=sa.service_id where s.server_id<>0 and sa.fqdn is not null" | tr ',' '\n' | sed -n 's#^[a-z]*://\([^/:]*\).*#FQDN=\1#p' | sort -u q "select a.fqdn from applications a join swarm_dockers sd on sd.id=a.destination_id where a.destination_type like '%SwarmDocker' and sd.server_id<>0 and a.fqdn is not null" | tr ',' '\n' | sed -n 's#^[a-z]*://\([^/:]*\).*#FQDN=\1#p' | sort -u mkdir -p "$RWD"; touch "$RWD/managed_known_hosts"; chmod 600 "$RWD/managed_known_hosts" q "select s.id, s.name, s.ip, coalesce(s.port,22), coalesce(nullif(s.\"user\",''),'root'), pk.uuid from servers s join private_keys pk on pk.id=s.private_key_id where s.id<>0 order by s.id" | while IFS=$'\x1f' read -r sid name ip port user keyuuid; do printf '%s' "$sid" | grep -Eqx '[0-9]+' || continue printf '%s' "$name" | grep -Eqx '[A-Za-z0-9][A-Za-z0-9_.-]{0,250}' || { emit MWARN "server $sid has an unsupported name"; continue; } printf '%s' "$ip" | grep -Eqx '[A-Za-z0-9.:-]{1,253}' || { emit MWARN "$name has an invalid address"; continue; } printf '%s' "$port" | grep -Eqx '[0-9]{1,5}' || { emit MWARN "$name has an invalid port"; continue; } printf '%s' "$user" | grep -Eqx '[a-z_][a-z0-9_-]{0,31}' || { emit MWARN "$name has an invalid user"; continue; } printf '%s' "$keyuuid" | grep -Eqx '[A-Za-z0-9_-]{1,64}' || { emit MWARN "$name has no usable SSH key"; continue; } key="/data/coolify/ssh/keys/ssh_key@$keyuuid" [ -f "$key" ] || { emit MWARN "$name SSH key is missing"; continue; } probe="$RWD/managed-probe-$sid-$$" if ! ssh -i "$key" -p "$port" -o BatchMode=yes -o StrictHostKeyChecking=accept-new \ -o UserKnownHostsFile="$RWD/managed_known_hosts" -o ConnectTimeout=12 -o LogLevel=ERROR -- "$user@$ip" \ 'if [ "$(id -u)" = 0 ]; then exec bash -s; else exec sudo -n bash -s; fi' >"$probe" 2>/dev/null <<'REMOTE' set -o pipefail one() { printf '%s' "$1" | tr -d '\000-\037\177' | head -c 1024; } printf 'ARCH|%s\n' "$(uname -m)" docker volume ls -q | while read -r v; do case "$v" in coolify-db|coolify-redis|buildx_buildkit_*) continue ;; esac users=$(docker ps -a --filter volume="$v" --format '{{.Names}}' | tr '\n' ','); users=${users%,} [ -n "$users" ] && ! printf '%s' "$users" | tr ',' '\n' | grep -qv -e '^coolify' -e '^buildx_buildkit' && continue mp=$(docker volume inspect -f '{{.Mountpoint}}' "$v") lb=$(docker volume inspect -f '{{range $k,$v := .Labels}}{{$k}}={{$v}}{{"\n"}}{{end}}' "$v" | base64 | tr -d '\n') printf 'VOL|%s|%s|%s|%s|%s|%s\n' "$(one "$v")" "$(du -sb "$mp" 2>/dev/null | cut -f1)" "${users:+1}" "$(one "$mp")" "$lb" "$(one "$users")" done ids=$(docker ps -aq --filter label=coolify.managed=true) if [ -n "$ids" ]; then for id in $ids; do n=$(docker inspect -f '{{.Name}}' "$id" | sed 's#^/##') docker inspect -f '{{range .Mounts}}{{if eq .Type "bind"}}{{.Source}}{{"\n"}}{{end}}{{end}}' "$id" | while read -r p; do [ -n "$p" ] && [ -e "$p" ] && printf 'BIND|%s|%s|%s\n' "$(one "$p")" "$(du -sb "$p" 2>/dev/null | cut -f1)" "$(one "$n")"; done done tmp=/tmp/cm-managed-imgids-$$ docker inspect -f '{{.Image}}' $ids | sort -u >"$tmp" docker images --no-trunc --format '{{.ID}} {{.Repository}}:{{.Tag}}' | grep -v '' | while read -r id ref; do grep -qx "$id" "$tmp" && printf 'IMG|%s|%s\n' "$(one "$ref")" "$(docker image inspect -f '{{.Size}}' "$ref")"; done rm -f "$tmp" fi docker ps --format '{{.Names}}|{{.Image}}' | sed 's/^/RUN|/' REMOTE then rm -f "$probe"; emit MWARN "$name is unreachable or lacks passwordless sudo"; continue fi remote=$(cat "$probe"); rm -f "$probe" arch=$(printf '%s\n' "$remote" | sed -n 's/^ARCH|//p' | head -n 1) printf '%s' "$arch" | grep -Eqx '[a-z0-9_]{1,16}' || { emit MWARN "$name returned an invalid architecture"; continue; } printf 'MSERVER=%s\x1f%s\x1f%s\x1f%s\x1f%s\x1f%s\x1f%s\n' "$sid" "$name" "$ip" "$port" "$user" "$keyuuid" "$arch" while IFS='|' read -r kind a b c d e f; do case "$kind" in VOL) printf 'MVOL=%s\x1f%s\x1f%s\x1f%s\x1f%s\x1f%s\x1f%s\x1f%s\n' "$sid" "$name" "$a" "$b" "$c" "$d" "$e" "$f" ;; BIND) printf 'MBIND=%s\x1f%s\x1f%s\x1f%s\x1f%s\n' "$sid" "$name" "$a" "$b" "$c" ;; IMG) printf 'MIMG=%s\x1f%s\x1f%s\x1f%s\n' "$sid" "$name" "$a" "$b" ;; RUN) printf 'MRUN=%s\x1f%s\x1f%s\x1f%s\n' "$sid" "$name" "$a" "$b" ;; esac done <<<"$remote" done true EOF def RS_DISCOVER_DST <<'EOF' emit ARCH "$(uname -m)"; . /etc/os-release 2>/dev/null; emit OS "${PRETTY_NAME:-unknown}" emit MID "$(cat /etc/machine-id 2>/dev/null)" emit IP "$(pubip)" mkdir -p /var/lib; emit FREE "$(df -Pk /var/lib | awk 'NR==2{printf "%.0f\n",$4*1024}')" emit INODES "$(df -Pi /var/lib | awk 'NR==2{print $4}')" emit CPU "$(getconf _NPROCESSORS_ONLN 2>/dev/null || nproc 2>/dev/null || echo 0)" emit MEMTOTAL "$(awk '/^MemTotal:/{printf "%.0f\n",$2*1024}' /proc/meminfo 2>/dev/null)" command -v docker >/dev/null && emit HASDOCKER 1 if command -v docker >/dev/null && docker container inspect coolify >/dev/null 2>&1; then img=$(docker container inspect -f '{{.Config.Image}}' coolify); emit COOLIFY "${img##*:}" emit COOLIFYUSERS "$(psqlc -c 'select count(*) from users' 2>/dev/null)" psqlc -F $'\x1f' -c "select uuid, translate(name, E'\\n\\r\\t\\x1f', ' ') from projects where team_id=0 order by name" 2>/dev/null | sed 's/^/DPROJECT=/' fi [ -d /data/coolify ] && emit DATA 1 [ -f "$RWD/journal" ] && emit JOURNAL "$(grep -c . "$RWD/journal")" true EOF def RS_PREPARE <<'EOF' mkdir -p "$RWD/dump" && chmod 700 "$RWD" pkg() { if command -v apt-get >/dev/null; then DEBIAN_FRONTEND=noninteractive apt-get update -qq && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq "$@" elif command -v dnf >/dev/null; then dnf install -y -q "$@" elif command -v yum >/dev/null; then yum install -y -q "$@" elif command -v zypper >/dev/null; then zypper -n -q in "$@" elif command -v apk >/dev/null; then apk add -q "$@" elif command -v pacman >/dev/null; then pacman -Sy --noconfirm "$@" fi } need=""; for t in rsync curl; do command -v $t >/dev/null || need="$need $t"; done command -v zstd >/dev/null || need="$need zstd" for t in timeout sha256sum; do command -v $t >/dev/null || need="$need coreutils"; done [ -n "$need" ] && { echo "installing:$need"; pkg $need /dev/null || { echo "missing $t and could not install it"; exit 1; }; done echo ready EOF # Records what the destination looked like before we touched it (first run only). def RS_JOURNAL_INIT <<'EOF' mkdir -p "$RWD" && chmod 700 "$RWD" if [ ! -f "$RWD/journal" ]; then journal "ENGINE $ENGINE" command -v docker >/dev/null && journal "DOCKER_PREEXISTED 1" || journal "DOCKER_PREEXISTED 0" { [ "$OVERWRITE" != 1 ] && command -v docker >/dev/null && docker container inspect coolify >/dev/null 2>&1; } && journal "COOLIFY_PREEXISTED 1" || journal "COOLIFY_PREEXISTED 0" [ -d /data/coolify ] && journal "DATA_PREEXISTED 1" || journal "DATA_PREEXISTED 0" fi cat "$RWD/journal" EOF def RS_KEYGEN <<'EOF' mkdir -p "$RWD" && chmod 700 "$RWD" [ -f "$RWD/id_ed25519" ] || ssh-keygen -q -t ed25519 -N '' -C "$MARKER_EPH" -f "$RWD/id_ed25519" /dev/null | cut -d: -f6) [ -n "$h" ] || h=$(awk -F: -v u="$TUSER" '$1==u{print $6}' /etc/passwd) [ -n "$h" ] && [ -d "$h" ] || { echo "user $TUSER not found"; exit 1; } mkdir -p "$h/.ssh"; touch "$h/.ssh/authorized_keys" line="$PUB"; [ -n "$OPTS" ] && line="$OPTS $PUB" grep -qF -- "$PUB" "$h/.ssh/authorized_keys" || printf '%s\n' "$line" >>"$h/.ssh/authorized_keys" chown -- "$TUSER" "$h/.ssh" "$h/.ssh/authorized_keys" 2>/dev/null chmod 700 "$h/.ssh"; chmod 600 "$h/.ssh/authorized_keys" [ -n "$JOURNAL_IT" ] && journal "AUTHKEY $TUSER $PUB" echo authorized EOF def RS_UNAUTHORIZE <<'EOF' homes | while read -r h; do f="$h/.ssh/authorized_keys" [ -f "$f" ] && grep -q -- "$MARKER" "$f" && sed -i "/$MARKER/d" "$f" done true EOF def RS_TEST_DIRECT <<'EOF' $DSSH -n -o ConnectTimeout=10 -- "$DT" true && echo DIRECT_OK EOF # Runs on SOURCE: rsync each record "label US src US dst US mode US volume" straight to the destination. # mode: "delete" mirrors (removes extra files), "freeze" pauses the containers using the volume during the copy. def RS_RSYNC <<'EOF' set -f args=(-aHS --numeric-ids --info=progress2 --no-inc-recursive --partial --partial-dir=.cm-partial -s -e "$DSSH") [ -n "$DS" ] && args+=(--rsync-path="$DS rsync") if [ "$ZIP" = 1 ]; then args+=(-z) if rsync --version 2>/dev/null | grep -qi zstd && $DSSH -n -- "$DT" "$DS rsync --version" 2>/dev/null | grep -qi zstd; then args+=(--compress-choice=zstd) fi fi for x in $EXCLUDES; do args+=(--exclude="$x"); done copy_one() ( label=$1 src=$2 dst=$3 mode=$4 vol=$5 paused="" r=0 extra=(); case "$mode" in *delete*) extra+=(--delete) ;; esac unfreeze() { [ -n "$paused" ] && docker unpause $paused >/dev/null 2>&1; paused=""; } trap unfreeze EXIT INT TERM HUP run_rsync() { if [ -n "$paused" ]; then timeout "$FREEZE_TIMEOUT" rsync "$@" || { r=$?; [ $r = 124 ] && echo "@@ freeze limit reached (${FREEZE_TIMEOUT}s); transfer aborted safely"; return $r; } else rsync "$@"; fi } case "$dst" in ''|/|/etc|/etc/*|/usr|/var|/var/lib|/var/lib/docker|/root|/home|/boot|/bin|/sbin|/lib|/lib64|/opt|/srv) echo "@@ refusing unsafe destination path '$dst' ($label)"; exit 1 ;; esac case "$src" in ''|/) echo "@@ refusing unsafe source path '$src' ($label)"; exit 1 ;; esac echo "@@ $label" case "$mode" in *freeze*) ids=$(docker ps -q --filter "volume=$vol" --filter status=running 2>/dev/null | tr '\n' ' ') if [ -n "${ids// /}" ]; then docker pause $ids >/dev/null 2>&1 && paused=$ids && echo "@@ froze $(echo $ids | wc -w) container(s), limit ${FREEZE_TIMEOUT}s" nohup sh -c "sleep $((FREEZE_TIMEOUT + 10)); docker unpause $ids" >/dev/null 2>&1 /dev/null 2>&1 && { echo "healthy on :$port"; exit 0; } [ $((i % 6)) = 0 ] && echo "waiting for Coolify ($((i * 5))s)" sleep 5 done exit 1 EOF def RS_CREATE_VOLUMES <<'EOF' while IFS='|' read -r name labels; do [ -z "$name" ] && continue printf '%s' "$name" | grep -Eqx '[A-Za-z0-9][A-Za-z0-9_.-]*' || { echo "skip invalid volume name"; continue; } if ! docker volume inspect "$name" >/dev/null 2>&1; then largs=() while IFS= read -r l; do [ -n "$l" ] && largs+=(--label "$l"); done <<<"$(printf '%s' "$labels" | base64 -d 2>/dev/null)" docker volume create "${largs[@]}" "$name" >/dev/null || { echo "failed $name"; exit 1; } journal "VOLUME $name" fi echo "MP=$name|$(docker volume inspect -f '{{.Mountpoint}}' "$name")" done <<<"$SPEC" EOF def RS_DUMP <<'EOF' mkdir -p "$RWD/dump" && chmod 700 "$RWD" "$RWD/dump" docker exec coolify-db pg_dump -U "$(dbu)" -d "$(dbn)" -Fc >"$RWD/dump/coolify.dump" && chmod 600 "$RWD/dump/coolify.dump" ls -lh "$RWD/dump/coolify.dump" | awk '{print "dump size:", $5}' EOF # DESTINATION (clone engine): restore the control plane from the dump. def RS_RESTORE <<'EOF' docker stop coolify coolify-realtime >/dev/null 2>&1 docker exec -i coolify-db pg_restore --clean --if-exists --no-acl --no-owner -U "$(dbu)" -d "$(dbn)" <"$RWD/dump/coolify.dump" 2>&1 | grep -E '^pg_restore: (error|warning)' | grep -v 'does not exist' | cut -c1-200 | tail -n 20 n=$(psqlc -c "select count(*) from servers" 2>/dev/null) [ "${n:-0}" -ge 1 ] || { echo "restore verification failed"; exit 1; } echo "restored: $n servers" [ "$FIXIP" = 1 ] && psqlc -c "update servers set ip='host.docker.internal' where id=0" >/dev/null && echo " localhost server ip -> host.docker.internal" if ip4 "$OLDIP" && ip4 "$NEWIP" && [ "$OLDIP" != "$NEWIP" ]; then psqli -v old="$OLDIP" -v new="$NEWIP" >/dev/null 2>&1 <<'SQL' && echo " instance public IP updated" update instance_settings set public_ipv4 = :'new' where public_ipv4 = :'old'; SQL fi # Copy mode: nothing scheduled may run twice. Pause what's enabled, remember it for --cutover. if [ "$PAUSE" = 1 ]; then : >"$RWD/paused-tasks"; chmod 600 "$RWD/paused-tasks" for t in scheduled_tasks scheduled_database_backups scheduled_volume_backups; do psqlc -c "select '$t ' || id from $t where enabled" 2>/dev/null >>"$RWD/paused-tasks" psqlc -c "update $t set enabled=false where enabled" >/dev/null 2>&1 done echo " paused $(grep -c . "$RWD/paused-tasks") scheduled tasks/backups until cutover" fi rm -rf /data/coolify/ssh/mux/* 2>/dev/null curl -fsSL "$URL" -o "$RWD/install.sh" || exit 1 if [ -n "$INSTALL_SHA256" ]; then actual=$(sha256sum "$RWD/install.sh" | awk '{print $1}') [ "$actual" = "$INSTALL_SHA256" ] || { echo "installer checksum mismatch"; exit 1; } fi bash "$RWD/install.sh" "$CV" /dev/null 2>&1 && n=$((n + 1)) done <"$RWD/paused-tasks" rm -f "$RWD/paused-tasks" echo "re-enabled $n scheduled tasks/backups" EOF # DESTINATION: run a PHP file inside the coolify container (file content in $PHP). def RS_PHP <<'EOF' mkdir -p "$RWD" && chmod 700 "$RWD" f="$RWD/run-$$.php"; printf '%s' "$PHP" >"$f"; chmod 600 "$f" trap 'rm -f "$f"; docker exec -u 0 coolify rm -f /tmp/cm-run.php /tmp/cm-in.json /tmp/cm-out.json >/dev/null 2>&1' EXIT owner="$(docker exec coolify id -u):$(docker exec coolify id -g)" # php runs as this user (www-data) give() { docker cp "$1" "coolify:$2" >/dev/null && docker exec -u 0 coolify sh -c "chown $owner $2 && chmod 600 $2"; } [ -n "$INFILE" ] && [ -f "$RWD/$INFILE" ] && { give "$RWD/$INFILE" /tmp/cm-in.json || exit 1; } give "$f" /tmp/cm-run.php || exit 1 docker exec -e CM_ARGS="$ARGS" -e CM_REUSE_IMAGES="$REUSE_IMAGES" -e OLDIP="$OLDIP" -e NEWIP="$NEWIP" \ -e TARGET_PROJECT_UUID="$TARGET_PROJECT_UUID" -e PROJECT_MAP_B64="$PROJECT_MAP_B64" coolify php /tmp/cm-run.php "$@" rc=$? if [ -n "$OUTFILE" ]; then docker cp "coolify:/tmp/cm-out.json" "$RWD/$OUTFILE" >/dev/null 2>&1 && chmod 600 "$RWD/$OUTFILE"; fi exit $rc EOF # SOURCE, at cutover: stop what moves (and Coolify itself for a full clone). Every change is recorded in rollback.sh. def RS_STOP_SOURCE <<'EOF' mkdir -p "$RWD" && chmod 700 "$RWD" rb="$RWD/rollback.sh" [ -f "$rb" ] || { echo '#!/usr/bin/env bash'; echo '# coolify-migrate: restores the OLD server to how it was before cutover.' echo 'docker rm -f cm-bridge >/dev/null 2>&1; rm -f /etc/cron.d/coolify-migrate-bridge'; } >"$rb" rec() { p=$(docker inspect -f '{{.HostConfig.RestartPolicy.Name}}' "$1" 2>/dev/null) || return 0 grep -q -- " $1; docker start $1\$" "$rb" || echo "docker update --restart=$p $1; docker start $1" >>"$rb"; } stop="" for n in $NAMES; do printf '%s' "$n" | grep -Eqx '[A-Za-z0-9][A-Za-z0-9_.-]*' || continue docker container inspect "$n" >/dev/null 2>&1 || continue rec "$n"; docker update --restart=no "$n" >/dev/null; stop="$stop $n" done [ -n "$stop" ] && docker stop -t 30 $stop >/dev/null && echo "stopped:$stop" # in parallel chmod 700 "$rb"; echo "rollback script: $rb" EOF # SOURCE CONTROL PLANE: stream a volume or bind directory from the managed # server that owns it. The stream is compressed on that host and contains no # shell-interpreted path data. def RS_MANAGED_TAR <<'EOF' printf '%s' "$SID" | grep -Eqx '[0-9]+' || exit 2 case "$PATH_TO_COPY" in /*/*) ;; *) echo "invalid managed source path" >&2; exit 2 ;; esac printf '%s' "$VOLUME" | grep -Eqx '[A-Za-z0-9_.-]*' || exit 2 row=$(psqlc -F $'\x1f' -c "select s.ip,coalesce(s.port,22),coalesce(nullif(s.\"user\",''),'root'),pk.uuid from servers s join private_keys pk on pk.id=s.private_key_id where s.id=$SID" 2>/dev/null) IFS=$'\x1f' read -r ip port user keyuuid <<<"$row" printf '%s' "$ip" | grep -Eqx '[A-Za-z0-9.:-]{1,253}' || exit 2 printf '%s' "$port" | grep -Eqx '[0-9]{1,5}' || exit 2 printf '%s' "$user" | grep -Eqx '[a-z_][a-z0-9_-]{0,31}' || exit 2 printf '%s' "$keyuuid" | grep -Eqx '[A-Za-z0-9_-]{1,64}' || exit 2 key="/data/coolify/ssh/keys/ssh_key@$keyuuid"; [ -f "$key" ] || exit 2 ssh -i "$key" -p "$port" -o BatchMode=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile="$RWD/managed_known_hosts" \ -o ConnectTimeout=12 -o LogLevel=ERROR -- "$user@$ip" \ "if [ \"\$(id -u)\" = 0 ]; then exec bash -s -- $(sq "$PATH_TO_COPY") $(sq "$VOLUME") $FREEZE $FREEZE_TIMEOUT; else exec sudo -n bash -s -- $(sq "$PATH_TO_COPY") $(sq "$VOLUME") $FREEZE $FREEZE_TIMEOUT; fi" <<'REMOTE' set -o pipefail path=$1 vol=$2 freeze=$3 limit=$4 paused="" case "$path" in /*/*) ;; *) exit 2 ;; esac case "$path" in /|/etc|/usr|/var|/var/lib|/var/lib/docker|/root|/home|/boot|/bin|/sbin|/lib|/lib64|/opt|/srv) exit 2 ;; esac unfreeze() { [ -n "$paused" ] && docker unpause $paused >/dev/null 2>&1; paused=""; } trap unfreeze EXIT INT TERM HUP if [ "$freeze" = 1 ] && [ -n "$vol" ]; then ids=$(docker ps -q --filter "volume=$vol" --filter status=running | tr '\n' ' ') if [ -n "${ids// /}" ]; then docker pause $ids >/dev/null && paused=$ids; fi fi timeout "$limit" tar -C "$path" --numeric-owner -cpf - . | gzip -1 rc=${PIPESTATUS[0]}; [ "$rc" = 124 ] && echo "freeze limit reached" >&2 exit "$rc" REMOTE EOF # SOURCE CONTROL PLANE: place the one-day transfer key and pinned destination # host keys on one managed server, then verify its direct route to destination. def RS_MANAGED_SETUP <<'EOF' printf '%s' "$SID" | grep -Eqx '[0-9]+' || exit 2 printf '%s' "$REACH" | grep -Eqx '[A-Za-z0-9.-]{1,253}' || exit 2 printf '%s' "$RPORT" | grep -Eqx '[0-9]{1,5}' || exit 2 printf '%s' "$DUSER" | grep -Eqx '[a-z_][a-z0-9_-]{0,31}' || exit 2 case "$DSUDO" in ''|sudo) ;; *) exit 2 ;; esac row=$(psqlc -F $'\x1f' -c "select s.ip,coalesce(s.port,22),coalesce(nullif(s.\"user\",''),'root'),pk.uuid from servers s join private_keys pk on pk.id=s.private_key_id where s.id=$SID" 2>/dev/null) IFS=$'\x1f' read -r ip port user keyuuid <<<"$row"; key="/data/coolify/ssh/keys/ssh_key@$keyuuid" [ -f "$key" ] && [ -f "$RWD/id_ed25519" ] && [ -f "$RWD/known_hosts" ] || exit 2 base=(ssh -i "$key" -p "$port" -o BatchMode=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile="$RWD/managed_known_hosts" -o ConnectTimeout=12 -o LogLevel=ERROR -- "$user@$ip") "${base[@]}" 'if [ "$(id -u)" = 0 ]; then exec bash -s; else exec sudo -n bash -s; fi' <<'REMOTE' || exit 1 set -o pipefail mkdir -p /var/lib/coolify-migrate; chmod 700 /var/lib/coolify-migrate pkg() { if command -v apt-get >/dev/null; then DEBIAN_FRONTEND=noninteractive apt-get update -qq && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq "$@" elif command -v dnf >/dev/null; then dnf install -y -q "$@" elif command -v yum >/dev/null; then yum install -y -q "$@" elif command -v apk >/dev/null; then apk add -q "$@" fi } need=""; command -v rsync >/dev/null || need="$need rsync"; command -v timeout >/dev/null || need="$need coreutils" [ -n "$need" ] && pkg $need /dev/null && command -v timeout >/dev/null REMOTE tar -C "$RWD" -cpf - id_ed25519 known_hosts | "${base[@]}" \ 'if [ "$(id -u)" = 0 ]; then tar -C /var/lib/coolify-migrate -xpf -; else sudo -n tar -C /var/lib/coolify-migrate -xpf -; fi' || exit 1 check="chmod 600 /var/lib/coolify-migrate/id_ed25519 /var/lib/coolify-migrate/known_hosts; ssh -n -i /var/lib/coolify-migrate/id_ed25519 -p $RPORT -o BatchMode=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile=/var/lib/coolify-migrate/known_hosts -o ConnectTimeout=12 -o LogLevel=ERROR -- $DUSER@$REACH $(sq "$DSUDO true")" "${base[@]}" "if [ \"\$(id -u)\" = 0 ]; then exec bash -c $(sq "$check"); else exec sudo -n bash -c $(sq "$check"); fi" && echo DIRECT EOF # SOURCE CONTROL PLANE: execute incremental rsync on the managed source host. # Freeze timing therefore covers only changed blocks, not a full relay copy. def RS_MANAGED_RSYNC <<'EOF' printf '%s' "$SID" | grep -Eqx '[0-9]+' || exit 2 case "$SRC_PATH" in /*/*) ;; *) exit 2 ;; esac case "$DST_PATH" in /*/*) ;; *) exit 2 ;; esac printf '%s' "$VOLUME" | grep -Eqx '[A-Za-z0-9_.-]*' || exit 2 printf '%s' "$REACH" | grep -Eqx '[A-Za-z0-9.-]{1,253}' || exit 2 printf '%s' "$RPORT" | grep -Eqx '[0-9]{1,5}' || exit 2 printf '%s' "$DUSER" | grep -Eqx '[a-z_][a-z0-9_-]{0,31}' || exit 2 case "$DSUDO" in ''|sudo) ;; *) exit 2 ;; esac row=$(psqlc -F $'\x1f' -c "select s.ip,coalesce(s.port,22),coalesce(nullif(s.\"user\",''),'root'),pk.uuid from servers s join private_keys pk on pk.id=s.private_key_id where s.id=$SID" 2>/dev/null) IFS=$'\x1f' read -r ip port user keyuuid <<<"$row"; key="/data/coolify/ssh/keys/ssh_key@$keyuuid"; [ -f "$key" ] || exit 2 ssh -i "$key" -p "$port" -o BatchMode=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile="$RWD/managed_known_hosts" \ -o ConnectTimeout=12 -o LogLevel=ERROR -- "$user@$ip" \ "if [ \"\$(id -u)\" = 0 ]; then exec bash -s -- $(sq "$SRC_PATH") $(sq "$DST_PATH") $(sq "$VOLUME") $FREEZE $FREEZE_TIMEOUT $(sq "$REACH") $RPORT $(sq "$DUSER") $(sq "$DSUDO") $ZIP; else exec sudo -n bash -s -- $(sq "$SRC_PATH") $(sq "$DST_PATH") $(sq "$VOLUME") $FREEZE $FREEZE_TIMEOUT $(sq "$REACH") $RPORT $(sq "$DUSER") $(sq "$DSUDO") $ZIP; fi" <<'REMOTE' set -o pipefail src=$1 dst=$2 vol=$3 freeze=$4 limit=$5 reach=$6 rport=$7 duser=$8 dsudo=$9 zip=${10} sq() { local s=${1//\'/\'\\\'\'}; printf "'%s'" "$s"; } case "$src:$dst" in /*/*:/*/*) ;; *) exit 2 ;; esac dssh="ssh -i /var/lib/coolify-migrate/id_ed25519 -p $rport -o BatchMode=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile=/var/lib/coolify-migrate/known_hosts -o ServerAliveInterval=15 -o LogLevel=ERROR" dt="$duser@$reach" args=(-aHS --numeric-ids --info=progress2 --no-inc-recursive --partial --partial-dir=.cm-partial -s --delete -e "$dssh") [ -n "$dsudo" ] && args+=(--rsync-path="$dsudo rsync") if [ "$zip" = 1 ]; then args+=(-z); fi paused="" unfreeze() { [ -n "$paused" ] && docker unpause $paused >/dev/null 2>&1; paused=""; } trap unfreeze EXIT INT TERM HUP if [ "$freeze" = 1 ] && [ -n "$vol" ]; then ids=$(docker ps -q --filter "volume=$vol" --filter status=running | tr '\n' ' ') if [ -n "${ids// /}" ]; then docker pause $ids >/dev/null && paused=$ids nohup sh -c "sleep $((limit + 10)); docker unpause $ids" >/dev/null 2>&1 0" 2>/dev/null | while IFS=$'\x1f' read -r ip port user keyuuid; do key="/data/coolify/ssh/keys/ssh_key@$keyuuid"; [ -f "$key" ] || continue ssh -n -i "$key" -p "$port" -o BatchMode=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile="$RWD/managed_known_hosts" \ -o ConnectTimeout=8 -o LogLevel=ERROR -- "$user@$ip" \ 'if [ "$(id -u)" = 0 ]; then rm -f /var/lib/coolify-migrate/id_ed25519 /var/lib/coolify-migrate/known_hosts; else sudo -n rm -f /var/lib/coolify-migrate/id_ed25519 /var/lib/coolify-migrate/known_hosts; fi' 2>/dev/null || true done true EOF # SOURCE CONTROL PLANE: stream selected images from one managed server. def RS_MANAGED_IMAGES <<'EOF' printf '%s' "$SID" | grep -Eqx '[0-9]+' || exit 2 refs=$(printf '%s' "$IMAGES_B64" | base64 -d 2>/dev/null) || exit 2 [ -n "$refs" ] || exit 0 for ref in $refs; do printf '%s' "$ref" | grep -Eqx '[a-z0-9][a-z0-9._/:@-]{0,254}' || exit 2; done row=$(psqlc -F $'\x1f' -c "select s.ip,coalesce(s.port,22),coalesce(nullif(s.\"user\",''),'root'),pk.uuid from servers s join private_keys pk on pk.id=s.private_key_id where s.id=$SID" 2>/dev/null) IFS=$'\x1f' read -r ip port user keyuuid <<<"$row"; key="/data/coolify/ssh/keys/ssh_key@$keyuuid" printf '%s' "$ip" | grep -Eqx '[A-Za-z0-9.:-]{1,253}' && printf '%s' "$port" | grep -Eqx '[0-9]{1,5}' && printf '%s' "$user" | grep -Eqx '[a-z_][a-z0-9_-]{0,31}' && [ -f "$key" ] || exit 2 ssh -i "$key" -p "$port" -o BatchMode=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile="$RWD/managed_known_hosts" \ -o ConnectTimeout=12 -o LogLevel=ERROR -- "$user@$ip" \ "if [ \"\$(id -u)\" = 0 ]; then exec bash -s -- $IMAGES_B64; else exec sudo -n bash -s -- $IMAGES_B64; fi" <<'REMOTE' set -o pipefail refs=$(printf '%s' "$1" | base64 -d) || exit 2 for ref in $refs; do printf '%s' "$ref" | grep -Eqx '[a-z0-9][a-z0-9._/:@-]{0,254}' || exit 2; done docker save $refs | gzip -1 REMOTE EOF # SOURCE CONTROL PLANE: stop managed-host workloads and create a rollback # script on the host before changing any restart policy. def RS_MANAGED_STOP <<'EOF' printf '%s' "$SID" | grep -Eqx '[0-9]+' || exit 2 uuids=$(printf '%s' "$NAMES_B64" | base64 -d 2>/dev/null) || exit 2 for u in $uuids; do printf '%s' "$u" | grep -Eqx '[A-Za-z0-9_-]{1,64}' || exit 2; done row=$(psqlc -F $'\x1f' -c "select s.ip,coalesce(s.port,22),coalesce(nullif(s.\"user\",''),'root'),pk.uuid from servers s join private_keys pk on pk.id=s.private_key_id where s.id=$SID" 2>/dev/null) IFS=$'\x1f' read -r ip port user keyuuid <<<"$row"; key="/data/coolify/ssh/keys/ssh_key@$keyuuid" [ -f "$key" ] || exit 2 ssh -i "$key" -p "$port" -o BatchMode=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile="$RWD/managed_known_hosts" \ -o ConnectTimeout=12 -o LogLevel=ERROR -- "$user@$ip" \ "if [ \"\$(id -u)\" = 0 ]; then exec bash -s -- $NAMES_B64; else exec sudo -n bash -s -- $NAMES_B64; fi" <<'REMOTE' set -o pipefail uuids=$(printf '%s' "$1" | base64 -d) || exit 2 names="" for u in $uuids; do printf '%s' "$u" | grep -Eqx '[A-Za-z0-9_-]{1,64}' || continue while read -r n; do printf '%s' "$n" | grep -Eqx '[A-Za-z0-9][A-Za-z0-9_.-]*' || continue case " $names " in *" $n "*) ;; *) names="$names $n" ;; esac done <<<"$(docker ps --format '{{.Names}}' | grep -F -- "$u" || true)" done mkdir -p /var/lib/coolify-migrate; chmod 700 /var/lib/coolify-migrate rb=/var/lib/coolify-migrate/rollback.sh printf '%s\n' '#!/usr/bin/env bash' '# Restores this source server after a failed consolidation.' >"$rb" stop="" for n in $names; do printf '%s' "$n" | grep -Eqx '[A-Za-z0-9][A-Za-z0-9_.-]*' || continue docker container inspect "$n" >/dev/null 2>&1 || continue p=$(docker inspect -f '{{.HostConfig.RestartPolicy.Name}}' "$n") printf 'docker update --restart=%q %q; docker start %q\n' "$p" "$n" "$n" >>"$rb" docker update --restart=no "$n" >/dev/null; stop="$stop $n" done [ -n "$stop" ] && docker stop -t 30 $stop >/dev/null chmod 700 "$rb"; echo "stopped $(echo $stop | wc -w)" REMOTE EOF def RS_MANAGED_ROLLBACK <<'EOF' printf '%s' "$SID" | grep -Eqx '[0-9]+' || exit 2 row=$(psqlc -F $'\x1f' -c "select s.ip,coalesce(s.port,22),coalesce(nullif(s.\"user\",''),'root'),pk.uuid from servers s join private_keys pk on pk.id=s.private_key_id where s.id=$SID" 2>/dev/null) IFS=$'\x1f' read -r ip port user keyuuid <<<"$row"; key="/data/coolify/ssh/keys/ssh_key@$keyuuid" [ -f "$key" ] || exit 2 ssh -n -i "$key" -p "$port" -o BatchMode=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile="$RWD/managed_known_hosts" \ -o ConnectTimeout=12 -o LogLevel=ERROR -- "$user@$ip" \ 'if [ "$(id -u)" = 0 ]; then bash /var/lib/coolify-migrate/rollback.sh; else sudo -n bash /var/lib/coolify-migrate/rollback.sh; fi' EOF # SOURCE: forward :80/:443 to the new server using the Traefik image Coolify already runs (no new images). def RS_BRIDGE <<'EOF' ip4 "$TO" || { echo "invalid target IP"; exit 1; } img=$(docker container inspect -f '{{.Config.Image}}' coolify-proxy 2>/dev/null) case "$img" in *traefik*) ;; *) echo "coolify-proxy is not Traefik ($img) - bridge unavailable"; exit 2 ;; esac mkdir -p "$RWD/bridge" && chmod 700 "$RWD/bridge" cat >"$RWD/bridge/dynamic.yml" </dev/null 2>&1; then grep -q ' coolify-proxy; docker start' "$rb" 2>/dev/null || echo "docker update --restart=$(docker inspect -f '{{.HostConfig.RestartPolicy.Name}}' coolify-proxy) coolify-proxy; docker start coolify-proxy" >>"$rb" docker update --restart=no coolify-proxy >/dev/null; docker stop coolify-proxy >/dev/null fi docker rm -f cm-bridge >/dev/null 2>&1 docker run -d --name cm-bridge --restart unless-stopped --network host -v "$RWD/bridge:/etc/cm-bridge:ro" "$img" \ --entrypoints.http.address=:80 --entrypoints.https.address=:443 --providers.file.filename=/etc/cm-bridge/dynamic.yml \ --log.level=ERROR >/dev/null || { echo "bridge failed to start"; docker start coolify-proxy >/dev/null 2>&1; exit 1; } echo "bridge :80/:443 -> $TO" # Self-removal: once every domain resolves to the new server for GRACE seconds (or after MAX), via cron, no docker.sock exposure. if [ -d /etc/cron.d ]; then date +%s >"$RWD/bridge/started" cat >"$RWD/bridge/reaper.sh" </dev/null || echo \$now); all=1 for d in $DOMAINS; do getent ahostsv4 "\$d" 2>/dev/null | awk '{print \$1}' | grep -qx "$TO" || all=0; done if [ "\$all" = 1 ]; then [ -f $RWD/bridge/since ] || echo \$now >$RWD/bridge/since; else rm -f $RWD/bridge/since; fi since=\$(cat $RWD/bridge/since 2>/dev/null || echo 0) if { [ "\$since" -gt 0 ] && [ \$((now - since)) -ge $GRACE ]; } || [ \$((now - start)) -ge $MAX ]; then docker rm -f cm-bridge >/dev/null 2>&1; rm -f /etc/cron.d/coolify-migrate-bridge fi R chmod 700 "$RWD/bridge/reaper.sh" printf '*/10 * * * * root /bin/sh %s/bridge/reaper.sh >/dev/null 2>&1\n' "$RWD" >/etc/cron.d/coolify-migrate-bridge chmod 644 /etc/cron.d/coolify-migrate-bridge echo "bridge removes itself once DNS points to $TO" fi EOF # SOURCE CONTROL PLANE: install the same temporary traffic bridge on a managed # source server so DNS for workloads hosted there can propagate safely. def RS_MANAGED_BRIDGE <<'EOF' printf '%s' "$SID" | grep -Eqx '[0-9]+' || exit 2 ip4 "$TO" || exit 2 printf '%s' "$BODY_B64" | grep -Eqx '[A-Za-z0-9+/=]+' || exit 2 printf '%s' "$DOMAINS_B64" | grep -Eqx '[A-Za-z0-9+/=]*' || exit 2 row=$(psqlc -F $'\x1f' -c "select s.ip,coalesce(s.port,22),coalesce(nullif(s.\"user\",''),'root'),pk.uuid from servers s join private_keys pk on pk.id=s.private_key_id where s.id=$SID" 2>/dev/null) IFS=$'\x1f' read -r ip port user keyuuid <<<"$row"; key="/data/coolify/ssh/keys/ssh_key@$keyuuid" [ -f "$key" ] || exit 2 ssh -i "$key" -p "$port" -o BatchMode=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile="$RWD/managed_known_hosts" \ -o ConnectTimeout=12 -o LogLevel=ERROR -- "$user@$ip" \ "if [ \"\$(id -u)\" = 0 ]; then exec bash -s -- $TO $DOMAINS_B64 $BODY_B64; else exec sudo -n bash -s -- $TO $DOMAINS_B64 $BODY_B64; fi" <<'REMOTE' set -o pipefail export TO=$1 DOMAINS="$(printf '%s' "$2" | base64 -d)" RWD=/var/lib/coolify-migrate GRACE=86400 MAX=604800 printf '%s' "$3" | base64 -d | bash REMOTE EOF # DESTINATION: can the new Coolify reach every remote server it manages? (clone engine) def RS_REMOTE_CHECK <<'EOF' command -v ssh >/dev/null || { echo "NOSSH=1"; exit 0; } mkdir -p "$RWD"; touch "$RWD/managed_known_hosts"; chmod 600 "$RWD/managed_known_hosts" priv() { case $1 in 10.*|192.168.*|172.1[6-9].*|172.2[0-9].*|172.3[01].*|100.6[4-9].*|100.[7-9][0-9].*|100.1[01][0-9].*|100.12[0-7].*) return 0 ;; esac; return 1; } psqlc -F $'\x1f' -c "select s.name, s.ip, s.port, s.\"user\", pk.uuid from servers s join private_keys pk on pk.id=s.private_key_id where s.id<>0" 2>/dev/null | while IFS=$'\x1f' read -r name ip port user uuid; do printf '%s' "$ip" | grep -Eqx '[A-Za-z0-9.:-]{1,253}' || continue printf '%s' "$port" | grep -Eqx '[0-9]{1,5}' || continue printf '%s' "$user" | grep -Eqx '[a-z_][a-z0-9_-]{0,31}' || continue printf '%s' "$uuid" | grep -Eqx '[A-Za-z0-9_-]{1,64}' || continue rip=$(getent hosts "$ip" 2>/dev/null | awk '{print $1; exit}'); rip=${rip:-$ip} sip=$(ip route get "$rip" 2>/dev/null | sed -n 's/.* src \([0-9.]*\).*/\1/p') { priv "$rip" && priv "$sip"; } || sip="" if timeout 20 ssh -n -i "/data/coolify/ssh/keys/ssh_key@$uuid" -p "$port" -o BatchMode=yes -o StrictHostKeyChecking=accept-new \ -o UserKnownHostsFile="$RWD/managed_known_hosts" -o ConnectTimeout=8 -o LogLevel=ERROR -- "$user@$ip" true 2>/dev/null; then st=OK; else st=FAIL; fi printf 'SRV=%s|%s|%s|%s|%s|%s|%s\n' "$(oneline "$name" | tr '|' '/')" "$ip" "$port" "$user" "$uuid" "$st" "$sip" done true EOF # SOURCE (still trusted by the remote servers): allow the new server's IP through host firewalls. def RS_REMOTE_OPEN <<'EOF' printf '%s\n' "$LIST" | while IFS='|' read -r name ip port user uuid allow; do [ -z "$ip" ] && continue ip4 "$allow" || continue printf '%s' "$port" | grep -Eqx '[0-9]{1,5}' || continue printf '%s' "$user" | grep -Eqx '[a-z_][a-z0-9_-]{0,31}' || continue printf '%s' "$uuid" | grep -Eqx '[A-Za-z0-9_-]{1,64}' || continue out=$(timeout 60 ssh -i "/data/coolify/ssh/keys/ssh_key@$uuid" -p "$port" -o BatchMode=yes -o StrictHostKeyChecking=yes \ -o UserKnownHostsFile="$RWD/managed_known_hosts" -o ConnectTimeout=8 -o LogLevel=ERROR -- "$user@$ip" sh -s -- "$allow" "$port" 2>&1 <<'R' ALLOW=$1; PORT=$2 S=""; [ "$(id -u)" = 0 ] || S="sudo -n" if command -v ufw >/dev/null 2>&1 && $S ufw status 2>/dev/null | grep -q "Status: active"; then $S ufw allow from "$ALLOW" to any port "$PORT" proto tcp comment coolify-migrate >/dev/null && echo ufw elif command -v firewall-cmd >/dev/null 2>&1 && $S firewall-cmd --state >/dev/null 2>&1; then $S firewall-cmd --permanent --add-rich-rule="rule family=ipv4 source address=$ALLOW port port=$PORT protocol=tcp accept" >/dev/null && $S firewall-cmd --reload >/dev/null && echo firewalld elif $S iptables -S INPUT 2>/dev/null | grep -qE '^-P INPUT (DROP|REJECT)|-j (DROP|REJECT)'; then $S iptables -I INPUT -s "$ALLOW" -p tcp --dport "$PORT" -m comment --comment coolify-migrate -j ACCEPT && { command -v netfilter-persistent >/dev/null 2>&1 && $S netfilter-persistent save >/dev/null 2>&1; echo iptables; } else echo none; fi R ) echo "OPEN=$(oneline "$name" | tr '|' '/')|$(printf '%s' "$out" | tail -n 1 | tr -cd 'a-z')" done true EOF # DESTINATION rollback for the clone engine (destination had no Coolify before): put it back to empty. def RS_WIPE_DEST <<'EOF' [ -f "$RWD/journal" ] || { echo "no journal - nothing recorded, refusing to wipe"; exit 1; } grep -qx 'COOLIFY_PREEXISTED 1' "$RWD/journal" && { echo "Coolify existed before - refusing to wipe"; exit 1; } echo "removing containers" ids=$(docker ps -aq --filter label=coolify.managed=true 2>/dev/null) [ -n "$ids" ] && docker rm -f $ids >/dev/null 2>&1 for c in coolify coolify-db coolify-redis coolify-realtime coolify-proxy coolify-sentinel; do docker rm -f "$c" >/dev/null 2>&1; done echo "removing volumes and networks" grep '^VOLUME ' "$RWD/journal" | cut -d' ' -f2 | while read -r v; do docker volume rm -f "$v" >/dev/null 2>&1; done docker volume rm -f coolify-db coolify-redis >/dev/null 2>&1 docker network rm coolify >/dev/null 2>&1 grep '^AUTHKEY ' "$RWD/journal" | while read -r _ u k; do h=$(getent passwd "$u" 2>/dev/null | cut -d: -f6); [ -f "$h/.ssh/authorized_keys" ] && grep -vF -- "$k" "$h/.ssh/authorized_keys" >"$h/.ssh/ak.tmp" && cat "$h/.ssh/ak.tmp" >"$h/.ssh/authorized_keys"; rm -f "$h/.ssh/ak.tmp" done # the installer adds its own localhost key for root: remove exactly that key for k in /data/coolify/ssh/keys/id.*@host.docker.internal.pub; do [ -f "$k" ] && [ -f /root/.ssh/authorized_keys ] && kb=$(awk '{print $2}' "$k") && [ -n "$kb" ] && grep -vF -- "$kb" /root/.ssh/authorized_keys >/root/.ssh/ak.tmp && cat /root/.ssh/ak.tmp >/root/.ssh/authorized_keys; rm -f /root/.ssh/ak.tmp done grep -qx 'DATA_PREEXISTED 0' "$RWD/journal" && rm -rf /data/coolify if grep -qx 'DOCKER_PREEXISTED 0' "$RWD/journal"; then echo "removing Docker (it was installed by the migration)" docker ps -aq | xargs -r docker rm -f >/dev/null 2>&1 if command -v apt-get >/dev/null; then DEBIAN_FRONTEND=noninteractive apt-get purge -y -qq docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin docker-ce-rootless-extras >/dev/null 2>&1 elif command -v dnf >/dev/null; then dnf remove -y -q docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin >/dev/null 2>&1 elif command -v yum >/dev/null; then yum remove -y -q docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin >/dev/null 2>&1 elif command -v apk >/dev/null; then apk del -q docker docker-cli-compose >/dev/null 2>&1 fi rm -rf /var/lib/docker /var/lib/containerd /etc/docker fi rm -rf "$RWD" echo "destination is empty again" EOF # DESTINATION rollback for the import engine: remove only what was added (rows, containers, volumes, dirs). def RS_UNDO_IMPORT_FILES <<'EOF' [ -f "$RWD/journal" ] || { echo "no journal"; exit 0; } grep '^UUID ' "$RWD/journal" | cut -d' ' -f2 | while read -r u; do printf '%s' "$u" | grep -Eqx '[A-Za-z0-9_-]{8,64}' || continue ids=$(docker ps -aq --filter "name=$u" 2>/dev/null); [ -n "$ids" ] && docker rm -f $ids >/dev/null 2>&1 && echo "removed containers of $u" for d in applications databases services; do grep -qx "DIR /data/coolify/$d/$u" "$RWD/journal" && rm -rf "/data/coolify/$d/$u" done done grep '^VOLUME ' "$RWD/journal" | cut -d' ' -f2 | while read -r v; do docker volume rm -f "$v" >/dev/null 2>&1 && echo "removed volume $v"; done grep '^AUTHKEY ' "$RWD/journal" | while read -r _ u k; do h=$(getent passwd "$u" 2>/dev/null | cut -d: -f6); [ -f "$h/.ssh/authorized_keys" ] && grep -vF -- "$k" "$h/.ssh/authorized_keys" >"$h/.ssh/ak.tmp" && cat "$h/.ssh/ak.tmp" >"$h/.ssh/authorized_keys"; rm -f "$h/.ssh/ak.tmp" done true EOF def RS_JOURNAL_ADD <<'EOF' printf '%s\n' "$LINES" | while IFS= read -r l; do [ -n "$l" ] && journal "$l"; done true EOF # Removes temporary files that may hold secrets. Runs on every exit, success or not. def RS_SCRUB <<'EOF' rm -rf "$RWD/dump" "$RWD/export.json" "$RWD"/run-*.php "$RWD/install.sh" "$RWD/imgids" rm -f "$RWD/id_ed25519" "$RWD/id_ed25519.pub" "$RWD/known_hosts" docker exec -u 0 coolify rm -f /tmp/cm-run.php /tmp/cm-in.json /tmp/cm-out.json >/dev/null 2>&1 true EOF # ------------------------------------------------- PHP (runs inside Coolify) -- # Shared bootstrap + helpers. Everything goes through Coolify's own models so # encrypted columns are decrypted/encrypted with the right APP_KEY. def PHP_HEAD <<'EOF' make(Illuminate\Contracts\Console\Kernel::class)->bootstrap(); use Illuminate\Support\Facades\{DB, Schema}; error_reporting(E_ALL & ~E_DEPRECATED); function cols(string $table): array { static $c = []; return $c[$table] ??= Schema::getColumnListing($table); } function say(string $s): void { echo preg_replace('/[\x00-\x09\x0b-\x1f\x7f]/', '', $s), "\n"; } const DB_CLASSES = ['StandalonePostgresql', 'StandaloneMysql', 'StandaloneMariadb', 'StandaloneMongodb', 'StandaloneRedis', 'StandaloneKeydb', 'StandaloneDragonfly', 'StandaloneClickhouse']; function db_classes(): array { return array_values(array_filter(array_map(fn ($c) => "App\\Models\\$c", DB_CLASSES), 'class_exists')); } function find_by_uuid(string $uuid) { foreach (array_merge(['App\\Models\\Application', 'App\\Models\\Service'], db_classes()) as $cls) { if ($m = $cls::where('uuid', $uuid)->first()) return $m; } return null; } function on_local($r): bool { if ($r instanceof App\Models\Service) return (int) $r->server_id === 0; return (int) data_get($r, 'destination.server.id', -1) === 0; } EOF # SOURCE: export resources (UUIDS, comma-separated, or ALL=1) with decrypted secrets -> /tmp/cm-out.json def PHP_EXPORT <<'EOF' function dump_model($m): ?array { if (! $m) return null; $cols = array_flip(cols($m->getTable())); $raw = array_intersect_key($m->getAttributes(), $cols); $enc = []; foreach ($m->getCasts() as $k => $cast) { if (is_string($cast) && str_starts_with($cast, 'encrypted') && array_key_exists($k, $raw)) { unset($raw[$k]); try { $enc[$k] = $m->getAttributeValue($k); } catch (Throwable $e) { $GLOBALS['warnings'][] = get_class($m)." #{$m->getKey()} $k could not be decrypted"; } } } return ['class' => get_class($m), 'table' => $m->getTable(), 'raw' => $raw, 'enc' => $enc]; } function dump_all($q): array { return $q->get()->map(fn ($m) => dump_model($m))->all(); } function morph($cls, $id, string $col = 'resource'): array { return ['type' => $cls, 'id' => $id, 'col' => $col]; } function children($m): array { $cls = get_class($m); $id = $m->id; $c = [ 'envs' => dump_all(App\Models\EnvironmentVariable::where('resourceable_type', $cls)->where('resourceable_id', $id)), 'pvols' => dump_all(App\Models\LocalPersistentVolume::where('resource_type', $cls)->where('resource_id', $id)), 'fvols' => dump_all(App\Models\LocalFileVolume::where('resource_type', $cls)->where('resource_id', $id)), 'backups'=> class_exists('App\\Models\\ScheduledDatabaseBackup') ? dump_all(App\Models\ScheduledDatabaseBackup::where('database_type', $cls)->where('database_id', $id)) : [], ]; return $c; } function private_key_dump($id) { return $id ? dump_model(App\Models\PrivateKey::find($id)) : null; } function export_resource($r): array { $env = $r->environment; $proj = $env?->project; $e = ['resource' => dump_model($r), 'environment' => dump_model($env), 'project' => dump_model($proj), 'children' => children($r)]; if ($r instanceof App\Models\Application) { $e['settings'] = dump_model(App\Models\ApplicationSetting::where('application_id', $r->id)->first()); $e['tasks'] = dump_all(App\Models\ScheduledTask::where('application_id', $r->id)); $e['private_key'] = private_key_dump($r->private_key_id); if ($r->source_type && $r->source_id && class_exists($r->source_type)) { $src = $r->source_type::find($r->source_id); $e['source'] = dump_model($src); $e['source_private_key'] = private_key_dump($src?->private_key_id); } } if ($r instanceof App\Models\Service) { $e['tasks'] = dump_all(App\Models\ScheduledTask::where('service_id', $r->id)); $e['sub'] = []; foreach ([App\Models\ServiceApplication::class, App\Models\ServiceDatabase::class] as $sc) { foreach ($sc::where('service_id', $r->id)->get() as $s) $e['sub'][] = ['model' => dump_model($s), 'children' => children($s)]; } } return $e; } $GLOBALS['warnings'] = []; $uuids = array_filter(explode(',', getenv('CM_ARGS') ?: '')); $all = in_array('ALL', $uuids, true); $list = []; foreach (array_merge(['App\\Models\\Application', 'App\\Models\\Service'], db_classes()) as $cls) { foreach ($cls::all() as $r) { if (! on_local($r)) continue; if ($all || in_array($r->uuid, $uuids, true)) $list[] = $r; } } $out = ['format' => 1, 'coolify' => config('constants.coolify.version'), 'resources' => [], 'warnings' => []]; foreach ($list as $r) { $out['resources'][] = export_resource($r); say("EXPORTED ".class_basename($r)." {$r->uuid} {$r->name}"); } $root = App\Models\User::find(0); if ($root) $out['root_user'] = ['raw' => array_intersect_key($root->getAttributes(), array_flip(['name', 'email', 'password', 'email_verified_at']))]; $out['warnings'] = $GLOBALS['warnings']; file_put_contents('/tmp/cm-out.json', json_encode($out, JSON_UNESCAPED_SLASHES)); chmod('/tmp/cm-out.json', 0600); foreach ($out['warnings'] as $w) say("WARN $w"); say("COUNT ".count($out['resources'])); EOF # DESTINATION: import an export file (/tmp/cm-in.json). Writes created rows to /tmp/cm-out.json for undo. # Env: OLDIP/NEWIP (rewrite), CM_ARGS="pause" to keep scheduled tasks/backups disabled until cutover. def PHP_IMPORT <<'EOF' $in = json_decode(file_get_contents('/tmp/cm-in.json'), true); $pause = str_contains(getenv('CM_ARGS') ?: '', 'pause'); $old = getenv('OLDIP') ?: ''; $new = getenv('NEWIP') ?: ''; $ipre = ($old && $new && $old !== $new && filter_var($old, FILTER_VALIDATE_IP) && filter_var($new, FILTER_VALIDATE_IP)) ? '/(?orderBy('id')->first(); if (! $dest) { say("FAIL no docker destination on the localhost server"); exit(1); } function rw($v) { global $ipre, $new; return ($ipre && is_string($v)) ? preg_replace($ipre, $new, $v) : $v; } function put(?array $d, array $over, bool $track = true) { global $created; if (! $d) return null; $cls = $d['class']; $m = new $cls; $cols = array_flip(cols($m->getTable())); $raw = $d['raw']; unset($raw['id']); foreach ($over as $k => $v) $raw[$k] = $v; foreach (['fqdn', 'docker_compose_raw', 'docker_compose', 'docker_compose_domains'] as $k) if (isset($raw[$k])) $raw[$k] = rw($raw[$k]); if (isset($raw['custom_labels']) && ($dec = base64_decode((string) $raw['custom_labels'], true)) !== false) $raw['custom_labels'] = base64_encode(rw($dec)); $m->setRawAttributes(array_intersect_key($raw, $cols)); foreach ($d['enc'] as $k => $v) if (isset($cols[$k])) $m->setAttribute($k, $k === 'value' ? rw($v) : $v); $m->exists = false; $m->saveQuietly(); if ($track) $created[] = [$m->getTable(), $m->getKey()]; return $m; } function by_uuid($cls, $uuid) { return $uuid ? $cls::where('uuid', $uuid)->first() : null; } function map_private_key(?array $d) { global $teamId; if (! $d) return null; if ($e = by_uuid(App\Models\PrivateKey::class, $d['raw']['uuid'] ?? null)) return $e->id; return put($d, ['team_id' => $teamId])->id; } function map_project(array $exp) { global $created, $teamId, $targetProjectUuid, $projectUuidMap; $p = $exp['project']; $e = $exp['environment']; $sourceProjectUuid = (string) ($p['raw']['uuid'] ?? ''); $selectedProjectUuid = $targetProjectUuid !== '' ? $targetProjectUuid : ($projectUuidMap[$sourceProjectUuid] ?? ''); if ($selectedProjectUuid !== '') { $proj = App\Models\Project::where('team_id', $teamId)->where('uuid', $selectedProjectUuid)->first(); if (! $proj) throw new RuntimeException("Selected destination project $selectedProjectUuid was not found"); } else { $proj = by_uuid(App\Models\Project::class, $p['raw']['uuid'] ?? null) ?? App\Models\Project::where('team_id', $teamId)->where('name', $p['raw']['name'])->first(); } if (! $proj) { // normal create: Coolify adds its project settings + default environment $proj = App\Models\Project::forceCreate(['name' => $p['raw']['name'], 'description' => $p['raw']['description'] ?? null, 'team_id' => $teamId, 'uuid' => $p['raw']['uuid']]); $created[] = ['projects', $proj->id, 'cascade']; } $env = $proj->environments()->where('name', $e['raw']['name'])->first(); if (! $env) { $env = put($e, ['project_id' => $proj->id]); } return $env; } function put_children(array $c, $owner, string $ownerClass) { global $pause, $paused, $teamId; foreach ($c['envs'] as $d) put($d, ['resourceable_type' => $ownerClass, 'resourceable_id' => $owner->id]); foreach ($c['pvols'] as $d) put($d, ['resource_type' => $ownerClass, 'resource_id' => $owner->id]); foreach ($c['fvols'] as $d) put($d, ['resource_type' => $ownerClass, 'resource_id' => $owner->id]); foreach ($c['backups'] as $d) { $was = (bool) ($d['raw']['enabled'] ?? false); $m = put($d, ['database_type' => $ownerClass, 'database_id' => $owner->id, 'team_id' => $teamId, 's3_storage_id' => null, 'save_s3' => false, 'enabled' => $pause ? false : $was]); if ($pause && $was) $paused[] = ['scheduled_database_backups', $m->id]; if (! empty($d['raw']['save_s3'])) say("WARN S3 upload for a backup of {$owner->name} was turned off - pick an S3 storage on the new server"); } } foreach ($in['resources'] as $exp) { $r = $exp['resource']; $cls = $r['class']; $uuid = $r['raw']['uuid']; if (find_by_uuid($uuid)) { say("SKIP ".class_basename($cls)." {$r['raw']['name']} ($uuid) already exists here"); continue; } try { DB::beginTransaction(); $env = map_project($exp); $over = ['environment_id' => $env->id]; if ($cls === 'App\\Models\\Service') { $over += ['server_id' => 0, 'destination_id' => $dest->id, 'destination_type' => get_class($dest)]; } else { $over += ['destination_id' => $dest->id, 'destination_type' => get_class($dest)]; } if (in_array('status', cols((new $cls)->getTable()))) $over['status'] = 'exited'; if ($cls === 'App\\Models\\Application') { $over['private_key_id'] = map_private_key($exp['private_key'] ?? null); if (! empty($exp['source'])) { $s = $exp['source']; if ((int) $s['raw']['id'] === 0) { $over['source_id'] = 0; } // public GitHub, exists everywhere elseif ($e = by_uuid($s['class'], $s['raw']['uuid'] ?? null)) { $over['source_id'] = $e->id; } else { $pk = map_private_key($exp['source_private_key'] ?? null); $over['source_id'] = put($s, ['team_id' => $teamId, 'private_key_id' => $pk])->id; } } } $m = put($r, $over); put_children($exp['children'], $m, $cls); if (! empty($exp['settings'])) put($exp['settings'], ['application_id' => $m->id]); foreach ($exp['tasks'] ?? [] as $d) { $was = (bool) ($d['raw']['enabled'] ?? false); $k = $cls === 'App\\Models\\Service' ? 'service_id' : 'application_id'; $t = put($d, [$k => $m->id, 'team_id' => $teamId, 'enabled' => $pause ? false : $was]); if ($pause && $was) $paused[] = ['scheduled_tasks', $t->id]; } foreach ($exp['sub'] ?? [] as $s) { $sm = put($s['model'], ['service_id' => $m->id] + (in_array('status', cols($s['model']['table'])) ? ['status' => 'exited'] : [])); put_children($s['children'], $sm, $s['model']['class']); } DB::commit(); say("IMPORTED ".class_basename($cls)." $uuid {$r['raw']['name']}"); } catch (Throwable $e) { DB::rollBack(); say("FAIL ".class_basename($cls)." {$r['raw']['name']}: ".$e->getMessage()); $GLOBALS['failed'] = true; } } // Fresh destination: let the old admin log in with the same email/password. if (getenv('CM_ARGS') && str_contains(getenv('CM_ARGS'), 'rootuser') && ! empty($in['root_user']) && App\Models\User::count() === 0) { $u = new App\Models\User(); $u->forceFill(['id' => 0] + $in['root_user']['raw']); $u->save(); if (! DB::table('team_user')->where(['team_id' => 0, 'user_id' => 0])->exists()) DB::table('team_user')->insert(['team_id' => 0, 'user_id' => 0, 'role' => 'owner', 'created_at' => now(), 'updated_at' => now()]); $created[] = ['users', 0]; say("ROOTUSER {$u->email}"); } file_put_contents('/tmp/cm-out.json', json_encode(['created' => $created, 'paused' => $paused])); exit(empty($GLOBALS['failed']) ? 0 : 3); EOF # DESTINATION: cancel every deployment created for imported applications before # their rows disappear. This mirrors Coolify's UI cancellation path and then # removes the migration-only queue history so undo leaves no orphan cards. def PHP_CANCEL_IMPORT_DEPLOYMENTS <<'EOF' $rows = json_decode(@file_get_contents('/tmp/cm-in.json') ?: '{}', true)['created'] ?? []; $applicationIds = []; foreach ($rows as $row) if (($row[0] ?? '') === 'applications' && is_numeric($row[1] ?? null)) $applicationIds[] = (int) $row[1]; $applicationIds = array_values(array_unique($applicationIds)); if ($applicationIds === []) { say('CANCELLED 0 imported deployments'); return; } $deployments = App\Models\ApplicationDeploymentQueue::whereIn('application_id', $applicationIds)->get(); $cancelled = App\Enums\ApplicationDeploymentStatus::CANCELLED_BY_USER->value; $serverIds = []; App\Models\ApplicationDeploymentQueue::whereIn('application_id', $applicationIds) ->whereIn('status', ['queued', 'in_progress'])->update(['status' => $cancelled]); foreach ($deployments as $deployment) { $uuid = (string) $deployment->deployment_uuid; $serverId = $deployment->build_server_id ?? $deployment->server_id; if (is_numeric($serverId)) $serverIds[(int) $serverId] = true; if (! preg_match('/^[A-Za-z0-9_-]{1,64}$/', $uuid)) continue; if ($server = App\Models\Server::find($serverId)) { try { instant_remote_process(["docker rm -f $uuid >/dev/null 2>&1 || true"], $server); } catch (Throwable $e) {} if (is_numeric($deployment->current_process_id)) { try { instant_remote_process(["kill -9 ".(int) $deployment->current_process_id." >/dev/null 2>&1 || true"], $server); } catch (Throwable $e) {} } } $deployment->update(['status' => $cancelled, 'current_process_id' => null]); } $n = DB::table('application_deployment_queues')->whereIn('application_id', $applicationIds)->delete(); foreach (array_keys($serverIds) as $serverId) if ($server = App\Models\Server::find($serverId)) { try { next_after_cancel($server); } catch (Throwable $e) { say("WARN queue advance on server $serverId failed: ".$e->getMessage()); } } say("CANCELLED $n imported deployments"); EOF # DESTINATION: repair leftovers from older undo versions. A queue row whose # application no longer exists cannot be a valid deployment, so this is scoped # tightly enough to run without the old migration journal. def PHP_CLEANUP_ORPHAN_DEPLOYMENTS <<'EOF' $ids = DB::table('application_deployment_queues') ->leftJoin('applications', 'applications.id', '=', 'application_deployment_queues.application_id') ->whereNull('applications.id') ->whereIn('application_deployment_queues.status', ['queued', 'in_progress']) ->pluck('application_deployment_queues.id')->all(); $deployments = App\Models\ApplicationDeploymentQueue::whereIn('id', $ids)->get(); $cancelled = App\Enums\ApplicationDeploymentStatus::CANCELLED_BY_USER->value; $serverIds = []; foreach ($deployments as $deployment) { $deployment->update(['status' => $cancelled]); $uuid = (string) $deployment->deployment_uuid; $serverId = $deployment->build_server_id ?? $deployment->server_id; if (is_numeric($serverId)) $serverIds[(int) $serverId] = true; if (preg_match('/^[A-Za-z0-9_-]{1,64}$/', $uuid) && ($server = App\Models\Server::find($serverId))) { try { instant_remote_process(["docker rm -f $uuid >/dev/null 2>&1 || true"], $server); } catch (Throwable $e) {} if (is_numeric($deployment->current_process_id)) { try { instant_remote_process(["kill -9 ".(int) $deployment->current_process_id." >/dev/null 2>&1 || true"], $server); } catch (Throwable $e) {} } } $deployment->update(['current_process_id' => null]); } $n = DB::table('application_deployment_queues')->whereIn('id', $ids)->delete(); foreach (array_keys($serverIds) as $serverId) if ($server = App\Models\Server::find($serverId)) { try { next_after_cancel($server); } catch (Throwable $e) { say("WARN queue advance on server $serverId failed: ".$e->getMessage()); } } say("CLEANED $n orphan deployments"); EOF # DESTINATION: undo an import - delete exactly the rows it created (newest first). def PHP_UNDO_IMPORT <<'EOF' $rows = json_decode(@file_get_contents('/tmp/cm-in.json') ?: '{}', true)['created'] ?? []; $n = 0; foreach (array_reverse($rows) as $row) { [$t, $id] = $row; $cascade = ($row[2] ?? '') === 'cascade'; if (! preg_match('/^[a-z_]+$/', $t)) continue; try { if ($cascade && $t === 'projects') { DB::table('project_settings')->where('project_id', $id)->delete(); DB::table('environments')->where('project_id', $id)->delete(); } if ($t === 'users') DB::table('team_user')->where('user_id', $id)->delete(); $n += DB::table($t)->where('id', $id)->delete(); } catch (Throwable $e) { say("WARN could not delete $t #$id: ".$e->getMessage()); } } say("REMOVED $n rows"); EOF # DESTINATION: start resources. CM_ARGS = comma list of uuids, "ALL", or "RUNNING:" (what ran on the old server). def PHP_START <<'EOF' $arg = getenv('CM_ARGS') ?: ''; $running = []; $all = false; $uuids = []; if (str_starts_with($arg, 'RUNNING:')) { foreach (explode(',', substr($arg, 8)) as $l) { [$n, $i] = array_pad(explode('|', $l, 2), 2, ''); if ($n !== '') $running[$n] = $i; } } elseif ($arg === 'ALL') { $all = true; } else { $uuids = array_filter(explode(',', $arg)); } $want = function (?string $uuid) use ($running, $all, $uuids) { if ($all) return true; if (! $uuid) return false; if ($uuids) return in_array($uuid, $uuids, true); foreach (array_keys($running) as $n) if (str_contains($n, $uuid)) return true; return false; }; $commitFor = function (string $uuid) use ($running) { foreach ($running as $i) if (str_starts_with($i, $uuid.':')) { $t = substr($i, strlen($uuid) + 1); if ($t !== '' && $t !== 'latest') return $t; } return null; }; $reuse = getenv('CM_REUSE_IMAGES') === '1'; try { App\Actions\Proxy\StartProxy::run(App\Models\Server::find(0)); say("OK proxy"); } catch (Throwable $e) { say("FAIL proxy (".$e->getMessage().")"); } foreach (db_classes() as $cls) foreach ($cls::all() as $db) { if (! on_local($db) || ! $want($db->uuid)) continue; try { App\Actions\Database\StartDatabase::run($db); say("OK database {$db->name}"); } catch (Throwable $e) { say("FAIL database {$db->name} (".$e->getMessage().")"); } } foreach (App\Models\Service::all() as $s) { if (! on_local($s) || ! $want($s->uuid)) continue; try { App\Actions\Service\StartService::run($s); say("OK service {$s->name}"); } catch (Throwable $e) { say("FAIL service {$s->name} (".$e->getMessage().")"); } } foreach (App\Models\Application::all() as $a) { if (! on_local($a) || ! $want($a->uuid)) continue; try { // A running container is the strongest evidence of the last successful // release. Fall back to Coolify's recorded successful commit, never a // newer branch HEAD unless neither value exists. $c = $commitFor($a->uuid) ?: ($a->git_commit_sha ?: null); $result = queue_application_deployment( application: $a, deployment_uuid: (string) Illuminate\Support\Str::uuid(), commit: $c ?? 'HEAD', force_rebuild: false, restart_only: $reuse ); $status = is_array($result) ? ($result['status'] ?? 'queued') : 'queued'; $ref = $c ? 'last successful commit '.substr($c, 0, 8) : 'HEAD (no successful commit recorded)'; $action = $reuse ? 'existing image requested; builds only if missing' : 'native build queued'; say("OK app {$a->name} ({$ref}; {$action}; {$status})"); } catch (Throwable $e) { say("FAIL app {$a->name} (".$e->getMessage().")"); } } EOF # DESTINATION: rewrite the old public IP everywhere Coolify stores it (clone engine), incl. encrypted env vars. def PHP_REWRITE_IP <<'EOF' $old = getenv('OLDIP'); $new = getenv('NEWIP'); if (! filter_var($old, FILTER_VALIDATE_IP) || ! filter_var($new, FILTER_VALIDATE_IP) || $old === $new) { say("REWROTE 0"); exit(0); } $re = '/(?{$f} ?? null; if ($f === 'custom_labels' && is_string($v)) { $d = base64_decode($v, true); if ($d !== false && ($n = $fix($d)) !== null) { $model->{$f} = base64_encode($n); $dirty = true; } continue; } if (($n = $fix($v)) !== null) { $model->{$f} = $n; $dirty = true; } } catch (Throwable $e) { } } if ($dirty) { $model->saveQuietly(); $count++; } }; foreach (['EnvironmentVariable', 'SharedEnvironmentVariable'] as $c) { $cls = "App\\Models\\$c"; if (class_exists($cls)) foreach ($cls::all() as $e) $touch($e, ['value']); } foreach (App\Models\Application::all() as $a) $touch($a, ['fqdn', 'docker_compose_domains', 'docker_compose_raw', 'docker_compose', 'custom_labels']); foreach (App\Models\Service::all() as $s) $touch($s, ['docker_compose_raw', 'docker_compose']); if (class_exists('App\\Models\\ServiceApplication')) foreach (App\Models\ServiceApplication::all() as $sa) $touch($sa, ['fqdn']); foreach (App\Models\ServerSetting::all() as $ss) $touch($ss, ['wildcard_domain']); foreach (App\Models\InstanceSettings::all() as $is) $touch($is, ['fqdn']); say("REWROTE $count"); EOF # DESTINATION: make sure the localhost server is usable on a freshly installed Coolify. def PHP_FRESH_SERVER <<'EOF' $s = App\Models\Server::find(0); if ($s) { if (! $s->user) { $s->user = 'root'; $s->saveQuietly(); } $s->settings?->update(['is_reachable' => true, 'is_usable' => true]); } say("OK"); EOF # DESTINATION: move every workload that referenced a managed source server onto # the new Coolify localhost destination. Source server rows remain until a # successful cutover so the copied dashboard retains a clear audit trail. def PHP_CONSOLIDATE <<'EOF' $dest = App\Models\StandaloneDocker::where('server_id', 0)->orderBy('id')->first(); if (! $dest) { say('FAIL no localhost Docker destination'); exit(1); } $moved = 0; $sourceServers = []; $move = function ($r) use ($dest, &$moved, &$sourceServers) { if (on_local($r)) return; $sid = $r instanceof App\Models\Service ? (int) $r->server_id : (int) data_get($r, 'destination.server.id', -1); if ($sid > 0) $sourceServers[$sid] = true; if ($r instanceof App\Models\Service) $r->server_id = 0; $r->destination_id = $dest->id; $r->destination_type = get_class($dest); if (in_array('status', cols($r->getTable()), true)) $r->status = 'exited'; $r->saveQuietly(); $moved++; }; foreach (App\Models\Application::all() as $r) $move($r); foreach (App\Models\Service::all() as $r) $move($r); foreach (db_classes() as $cls) foreach ($cls::all() as $r) $move($r); foreach (array_keys($sourceServers) as $sid) { if ($s = App\Models\Server::find($sid)) { $base = preg_replace('/ \(migrated to localhost\)$/', '', (string) $s->name); $s->name = $base.' (migrated to localhost)'; $s->saveQuietly(); $s->settings?->update(['is_reachable' => false, 'is_usable' => false]); } } say("CONSOLIDATED $moved resources from ".count($sourceServers)." managed servers"); EOF # DESTINATION: after a successful cutover, remove the now-unused managed-server # records from the new dashboard. The original Coolify database is untouched. def PHP_FINALIZE_CONSOLIDATION <<'EOF' $removed = 0; $kept = 0; foreach (App\Models\Server::where('id', '<>', 0)->get() as $s) { $busy = App\Models\Service::where('server_id', $s->id)->exists(); foreach (App\Models\StandaloneDocker::where('server_id', $s->id)->get() as $d) { if (App\Models\Application::where('destination_id', $d->id)->where('destination_type', get_class($d))->exists()) $busy = true; foreach (db_classes() as $cls) if ($cls::where('destination_id', $d->id)->where('destination_type', get_class($d))->exists()) $busy = true; } if ($busy) { say("WARN server {$s->name} still owns resources and was retained"); $kept++; continue; } try { App\Models\StandaloneDocker::where('server_id', $s->id)->delete(); $s->delete(); $removed++; } catch (Throwable $e) { say("WARN could not remove server {$s->name}: ".$e->getMessage()); $kept++; } } say("FINALIZED $removed managed servers; retained $kept"); EOF default_key() { local k for k in id_ed25519 id_ecdsa id_rsa; do [[ -f $HOME/.ssh/$k ]] && { echo "$HOME/.ssh/$k"; return; }; done echo "$HOME/.ssh/id_ed25519" } # Temporarily allow passwordless sudo (validated with visudo, removed on exit). grant_sudo() { # role password local r=$1 pw=$2 u u=$(rssh "$r" 'id -un' 2>>"$LOG") local inner='printf "%s\n" "$L" >"$F.tmp" && chmod 440 "$F.tmp" && visudo -cf "$F.tmp" >/dev/null && mv "$F.tmp" "$F"' printf '%s\n' "$pw" | rssh "$r" "sudo -S -p '' env F=$(sq "$SUDOERS_FILE") L=$(sq "$u ALL=(ALL) NOPASSWD: ALL") sh -c $(sq "$inner")" >>"$LOG" 2>&1 && rssh "$r" 'sudo -n true' >>"$LOG" 2>&1 || return 1 setv SUDO_GRANT "$r" 1 } my_public_ip() { if [[ -z ${MY_IP:-} ]]; then MY_IP=$(curl -4fsS --max-time 5 https://ifconfig.io 2>/dev/null | head -c 64 | tr -d '\r\n '); is_ip4 "$MY_IP" || MY_IP=""; fi printf '%s' "${MY_IP:-this server}" } # OpenSSH expiry-time in UTC, N hours from now (GNU and BSD date). expiry_utc_hours() { local t=$(($(date +%s) + $1 * 3600)); date -u -d "@$t" +%Y%m%d%H%M%SZ 2>/dev/null || date -u -r "$t" +%Y%m%d%H%M%SZ; } # Highlighted copy-paste block ui_cmdbox() { # title lines... printf '\n %s%s%s\n' "$B" "$1" "$R"; shift local l; for l in "$@"; do printf '\n %s%s%s\n' "$GRN" "$l" "$R"; done; echo } where_hint() { info "Run it on the ${B}new server${R} ($(hv DST HOST)) — e.g. your cloud provider's web console" info "(AWS: EC2 ${G_ARR} Instances ${G_ARR} Connect ${G_ARR} EC2 Instance Connect) or an SSH session from your laptop." } authorized_keys_cmd() { # user pubkey (restricted, expires automatically) local line="restrict,expiry-time=\"$(expiry_utc_hours "$KEY_TTL_HOURS")\" $2" if [[ $1 == root ]]; then printf "sudo mkdir -p /root/.ssh && echo '%s' | sudo tee -a /root/.ssh/authorized_keys >/dev/null && sudo chmod 700 /root/.ssh && sudo chmod 600 /root/.ssh/authorized_keys" "$line" else printf "mkdir -p ~/.ssh && echo '%s' >> ~/.ssh/authorized_keys && chmod 700 ~/.ssh && chmod 600 ~/.ssh/authorized_keys" "$line" fi } guide_new_key() { # role local r=$1 u __x ttl u=$(hv "$r" USER) mkdir -p "$HOME/.ssh" && chmod 700 "$HOME/.ssh" [[ -f $SETUP_KEY ]] || ssh-keygen -q -t ed25519 -N '' -C "$SETUP_MARKER@$(hostname)" -f "$SETUP_KEY" = 1 && 10#$ttl <= 720)) && { KEY_TTL_HOURS=$((10#$ttl)); break; } warn "Enter 1-720 hours. The default is 24." done fi ui_cmdbox "Copy this whole line and run it once on the new server$([[ $u != root ]] && printf ' as %s' "$u"):" \ "$(authorized_keys_cmd "$u" "$(awk -v m="$SETUP_MARKER" '{print $1" "$2" "m}' "$SETUP_KEY.pub")")" where_hint [[ $u == root ]] && info "Works on AWS/GCP too: root accepts keys even when root password login is off." info "The key only works for $KEY_TTL_HOURS hour(s) and is removed from the new server when this run ends." ui_ask __x "Press Enter once you've run it" } paste_key() { # role local r=$1 f="$HOME/.ssh/coolify_migrate_pasted" line n=0 mkdir -p "$HOME/.ssh" && chmod 700 "$HOME/.ssh" info "Paste the private key now — everything from ${B}-----BEGIN${R} to ${B}-----END … PRIVATE KEY-----${R}." info "${D}(Hidden while you paste. It's saved with mode 600 on this server only.)${R}" ( umask 077; : >"$f" ) while IFS= read -rs line <"$TTY"; do line=${line%$'\r'}; line=${line#$'\e[200~'}; line=${line%$'\e[201~'} [[ -z $line && $n -eq 0 ]] && continue printf '%s\n' "$line" >>"$f"; n=$((n + 1)) [[ $line == *END*PRIVATE\ KEY----- ]] && break done if ssh-keygen -y -P '' -f "$f" >/dev/null 2>&1; then ok "Key received ($n lines) ${G_ARR} $f"; setv "$r" KEY "$f" [[ $(hv "$r" USER) == root ]] && info "Cloud keys usually log in as ${B}ubuntu${R} (Ubuntu), ${B}ec2-user${R} (Amazon Linux) or ${B}admin${R} (Debian) — the script will switch automatically if the server asks." else rm -f "$f"; err "That isn't a valid, unencrypted private key. (Passphrase-protected? Use 'Use a key file' instead.)"; return 1 fi } guide_password() { # role local r=$1 warn "Most cloud servers (AWS, GCP, Azure, many VPS images) have SSH password login turned ${B}off${R}." ui_cmdbox "If yours does, enable it on the new server first (one line, run as root or a sudo user):" \ "printf 'PasswordAuthentication yes\nKbdInteractiveAuthentication yes\nPermitRootLogin yes\n' | sudo tee $PWCONF >/dev/null && (sudo systemctl restart ssh || sudo systemctl restart sshd)" \ "sudo passwd $(hv "$r" USER) # only if $(hv "$r" USER) has no password yet" where_hint info "That setting is removed again automatically when this run ends." info "Tip: ${B}Create a new key for me${R} is simpler and safer than passwords." } guide_agent() { local k found="" for k in "$HOME"/.ssh/*.pub; do [[ -f $k ]] && found+=" $(basename "${k%.pub}")"; done if [[ -n $found ]]; then info "Will try the keys in ~/.ssh:${B}$found${R} (and any loaded in ssh-agent)." else warn "No SSH keys found in ~/.ssh on this machine — choose ${B}Create a new key for me${R} instead."; fi } # Explain a failed connection in plain words + what to do about it diagnose_ssh() { # role errfile local r=$1 e host port user local methods e=$(cat "$2" 2>/dev/null); host=$(hv "$r" HOST); port=$(hv "$r" PORT); user=$(hv "$r" USER) methods=$(printf '%s' "$e" | sed -n 's/.*Permission denied (\([^)]*\)).*/\1/p' | head -n 1) case $e in *"Permission denied"*) if [[ -n $methods && $methods != *password* && $(hv "$r" AUTH) == password ]]; then warn "This server has password login ${B}disabled${R} — it only accepts SSH keys." info "Either enable passwords (commands shown above) or pick ${B}Create a new key for me${R}." elif [[ -n $methods && $methods != *password* ]]; then warn "The server only accepts SSH keys, and it doesn't trust this one (yet)." info "Fix: choose ${B}Create a new key for me${R} and run the line it shows on the new server." else warn "Login refused — wrong password/key, or ${B}$user${R} isn't allowed to log in." fi [[ $user == root ]] && info "Cloud images often block root. Try the default user: ${B}ubuntu${R}@$host · ${B}ec2-user${R}@$host · ${B}admin${R}@$host — the script uses sudo." ;; *"Connection refused"*) warn "Nothing answered on $host:$port. Check the SSH port (default 22) and that the server is fully booted." ;; *"timed out"*|*"No route"*|*"unreachable"*) warn "Can't reach $host:$port — almost always a firewall / security group." info "Allow inbound ${B}TCP $port${R} from ${B}$(my_public_ip)${R} (AWS: EC2 ${G_ARR} Security Groups ${G_ARR} Inbound rules ${G_ARR} SSH)." ;; *"Could not resolve"*) warn "\"$host\" doesn't resolve. Check for typos, or use the server's IP address." ;; *"IDENTIFICATION HAS CHANGED"*|*"Host key verification failed"*) warn "The new server's fingerprint changed since this machine last saw it (reinstalled?)." ui_cmdbox "If that's expected, clear the old fingerprint and retry:" "ssh-keygen -R '$host'; ssh-keygen -R '[$host]:$port'" ;; esac } connect_role() { # role label local r=$1 label=$2 tgt idx key pass p probe nu re errf="$CTL_DIR/ssh-err" if is_local "$r"; then ok "Source: ${B}this server${R} ($(hostname))" else while :; do if [[ -z $(hv "$r" HOST) ]]; then ((INTERACTIVE)) || die "$label server not given (use --$(echo "$r" | tr A-Z a-z) user@host)" ui_ask tgt "$label server ${D}(user@host — port 22 unless you add :port)${R}${B}" [[ -z $tgt ]] && continue [[ $tgt == *@* ]] || info "No user given ${G_ARR} using ${B}root${R}" parse_target "$r" "$tgt" fi if [[ -z $(hv "$r" AUTH) ]]; then ui_select idx "How should I log in to $(target "$r") (port $(hv "$r" PORT))?" \ "Create a new key for me ${D}(recommended — I'll show you the one line to run on the new server)${R}" \ "Paste a private key ${D}(e.g. the .pem from AWS / your cloud provider)${R}" \ "Use a key file already on this machine" \ "Password" \ "Keys already set up here ${D}(~/.ssh, ssh-agent)${R}" case $idx in 0) setv "$r" AUTH key; guide_new_key "$r" ;; 1) setv "$r" AUTH key; paste_key "$r" || { setv "$r" AUTH ""; continue; } ;; 2) setv "$r" AUTH key ;; 3) setv "$r" AUTH password; guide_password "$r"; setv PWGUIDE "$r" 1 ;; *) setv "$r" AUTH agent; guide_agent ;; esac fi case $(hv "$r" AUTH) in key) if [[ -z $(hv "$r" KEY) ]]; then ui_ask key "Private key path" "$(default_key)"; setv "$r" KEY "${key/#\~/$HOME}"; fi if [[ ! -f $(hv "$r" KEY) ]]; then err "Key not found: $(hv "$r" KEY)" info "In a web terminal it's easier to choose ${B}Paste a private key${R}." setv "$r" KEY ""; setv "$r" AUTH ""; continue fi chmod 600 "$(hv "$r" KEY)" 2>/dev/null ;; password) password_supported || die "Password auth needs 'sshpass' or OpenSSH ${G_GE} 8.4. Install: apt install sshpass (or brew install sshpass)" if [[ -z $(hv "$r" PASS) ]]; then ((INTERACTIVE)) || die "No password for $label (set ${r}_PASSWORD env var)" ui_secret pass "Password for $(target "$r")"; setv "$r" PASS "$pass" fi ;; esac is_port "$(hv "$r" PORT)" || { err "Invalid port $(hv "$r" PORT)"; setv "$r" HOST ""; continue; } confirm_host_key "$r" if ui_run "Connecting to $label $(target "$r"):$(hv "$r" PORT)" open_master_logged "$r" "$errf"; then # Some clouds accept the key but force a "login as user X" message for root probe=$(rssh "$r" 'echo CM_PROBE_OK' 2>&1) re='login as the user "?([A-Za-z0-9_.-]+)' if [[ $probe != *CM_PROBE_OK* && $probe =~ $re ]]; then nu=${BASH_REMATCH[1]} warn "$(hv "$r" HOST) doesn't allow ${B}$(hv "$r" USER)${R} — it asks for ${B}$nu${R}. Switching to $nu@$(hv "$r" HOST)." close_master "$r"; setv "$r" USER "$nu"; continue fi ok "Connected to $label ${B}$(target "$r")${R} (port $(hv "$r" PORT))"; break fi cat "$errf" >>"$LOG" err "Could not connect to $(target "$r"):$(hv "$r" PORT)" grep -vE '^\s*$|Permanently added' "$errf" | tail -n 2 | sed 's/^/ /' diagnose_ssh "$r" "$errf" ((INTERACTIVE)) || die "Connection to $label failed" ui_select idx "What now?" "Retry ${D}(I fixed it)${R}" "Try a different login method" "Change server / user / port" "Abort" case $idx in 0) ;; 1) setv "$r" AUTH ""; setv "$r" KEY ""; setv "$r" PASS "" ;; 2) setv "$r" HOST ""; setv "$r" AUTH ""; setv "$r" KEY ""; setv "$r" PASS "" ;; *) die "Aborted" ;; esac done fi p=$(rssh "$r" 'if [ "$(id -u)" = 0 ]; then echo root; elif sudo -n true 2>/dev/null; then echo sudo; else echo none; fi' 2>>"$LOG") case $p in root) setv "$r" SUDO "" ;; sudo) setv "$r" SUDO "sudo"; info "Using passwordless sudo on $label" # a previous interrupted run may have left our temporary sudoers file: make sure it's removed this time rssh "$r" "sudo -n test -f $SUDOERS_FILE" 2>/dev/null && setv SUDO_GRANT "$r" 1 ;; *) warn "$(hv "$r" USER) on $label needs a password for sudo." local spw="" envn="${r}_SUDO_PASSWORD" spw=${!envn:-} [[ -z $spw && $(hv "$r" AUTH) == password ]] && spw=$(hv "$r" PASS) if [[ -z $spw ]] || ! grant_sudo "$r" "$spw"; then ((INTERACTIVE)) || die "sudo on $label needs a password: set $envn or run interactively" ui_secret spw "sudo password for $(hv "$r" USER) on $label" grant_sudo "$r" "$spw" || die "sudo rejected the password on $label" fi spw="" setv "$r" SUDO "sudo"; ok "Temporary passwordless sudo on $label ${D}(removed when done)${R}" ;; esac } # ---------------------------------------------------------------- cleanup --- # Runs on every exit. On a failed copy/cutover it rolls back automatically (unless --no-auto-rollback). TRUST_SET=0 cleanup() { local rc=$? r trap - EXIT tput cnorm 2>/dev/null ((HANDED_OFF)) && exit $rc # the background worker owns connections & cleanup now if ((EXECUTING && rc != 0)); then if ((AUTO_ROLLBACK)); then auto_rollback; else warn "Auto-rollback disabled. Undo later with: $(self_cmd) --undo"; fi fi # temporary access & secret-bearing files: always removed rscript DST "$RS_UNAUTHORIZE" MARKER="$KEY_MARKER" >/dev/null 2>&1 ((CONSOLIDATE)) && rscript SRC "$RS_MANAGED_SCRUB" >/dev/null 2>&1 rscript SRC "$RS_SCRUB" >/dev/null 2>&1 rscript DST "$RS_SCRUB" >/dev/null 2>&1 for r in SRC DST; do is_local "$r" && continue (($(hv PWGUIDE "$r"))) && rscript "$r" "[ -f $PWCONF ] && rm -f $PWCONF && (systemctl reload ssh || systemctl reload sshd || service ssh reload) >/dev/null 2>&1; true" >/dev/null 2>&1 done for r in SRC DST; do (($(hv SUDO_GRANT "$r"))) && rscript "$r" "rm -f $SUDOERS_FILE" >/dev/null 2>&1 done close_master SRC; close_master DST rm -rf "$CTL_DIR" [[ -n $WORKER_RC_FILE ]] && echo "$rc" >"$WORKER_RC_FILE" exit $rc } trap cleanup EXIT trap 'printf "\n"; err "Interrupted."; exit 130' INT TERM HUP # ------------------------------------------------------------ saved state --- STATE_FILE="" PLAN_FILE="" state_path() { local id="${DST_USER:-root}@${DST_HOST:-unknown}:${DST_PORT:-22}" printf '%s/state/%s' "$STATE_DIR" "$(printf '%s' "$id" | tr -c 'a-zA-Z0-9.@-' '_')" } legacy_state_path() { printf '%s/state/%s' "$STATE_DIR" "$(printf '%s' "$DST_HOST" | tr -c 'a-zA-Z0-9.-' '_')"; } done_step() { grep -qx "STEP $1" "$STATE_FILE" 2>/dev/null; } mark() { echo "STEP $1" >>"$STATE_FILE"; } # Plan + connection details (never secrets) so --cutover / --undo can pick up later. # Plans are parsed data, never sourced as shell code. plan_b64() { printf '%s' "$1" | base64 | tr -d '\r\n'; } plan_unb64() { printf '%s' "$1" | base64 -d 2>/dev/null; } plan_scalar_allowed() { case $1 in ENGINE|DST_HOST|DST_USER|DST_PORT|DST_AUTH|DST_KEY|DST_HOST_KEY|SRC_LOCAL|SRC_HOST|SRC_USER|SRC_PORT|SRC_AUTH|SRC_KEY|SRC_HOST_KEY|S_IP|D_IP|S_VERSION|S_DBU|S_DBN|S_ARCH|D_ARCH|SEL_UUIDS|OPT_VOLUMES|OPT_BINDS|OPT_IMAGES|OPT_START|OPT_IPREWRITE|OPT_COMPRESS|OPT_BRIDGE|MODE|DST_REACH|S_LOCAL_PUB|S_LOCAL_USER|S_DOCKERCFG|INSTALL_SHA256|FREEZE_TIMEOUT|TRANSFER_JOBS|CONSOLIDATE|KEY_TTL_HOURS|TARGET_PROJECT_UUID) return 0 ;; *) return 1 ;; esac } plan_scalar_valid() { local key=$1 value=$2 item case $key in ENGINE) [[ $value == clone || $value == import ]] ;; DST_HOST|SRC_HOST) [[ $value =~ ^[A-Za-z0-9.:-]{1,253}$ ]] ;; DST_USER|SRC_USER|S_LOCAL_USER) is_user "$value" ;; DST_PORT|SRC_PORT) is_port "$value" ;; DST_AUTH|SRC_AUTH) [[ $value =~ ^(key|agent|password|local)$ ]] ;; DST_KEY|SRC_KEY) [[ -z $value || ( $value == /* && $value != *$'\n'* ) ]] ;; DST_HOST_KEY|SRC_HOST_KEY) [[ -z $value || $value =~ ^SHA256:[A-Za-z0-9+/=]+$ ]] ;; SRC_LOCAL|OPT_VOLUMES|OPT_BINDS|OPT_IMAGES|OPT_START|OPT_IPREWRITE|OPT_COMPRESS|OPT_BRIDGE|CONSOLIDATE) [[ $value =~ ^[01]$ ]] ;; S_IP|D_IP) [[ -z $value ]] || is_ip4 "$value" ;; S_VERSION) is_version "$value" ;; S_DBU|S_DBN) is_name "$value" ;; S_ARCH|D_ARCH) [[ $value =~ ^[a-z0-9_]{1,16}$ ]] ;; SEL_UUIDS) [[ -z $value || $value == ALL ]] && return 0 for item in ${value//,/ }; do is_uuid "$item" || return 1; done ;; MODE) [[ $value == direct || $value == relay ]] ;; DST_REACH) [[ -z $value || ( ${#value} -le 260 && $value =~ ^[A-Za-z0-9.:-]+$ ) ]] ;; S_LOCAL_PUB) [[ -z $value || $value =~ ^(ssh-ed25519|ssh-rsa|ecdsa-sha2-nistp[0-9]+)\ [A-Za-z0-9+/=]+(\ [^\'\"]{0,200})?$ ]] ;; S_DOCKERCFG) [[ -z $value ]] || is_abspath "$value" ;; INSTALL_SHA256) [[ -z $value || $value =~ ^[a-f0-9]{64}$ ]] ;; FREEZE_TIMEOUT) [[ $value =~ ^[0-9]+$ ]] && ((10#$value >= 5 && 10#$value <= 900)) ;; TRANSFER_JOBS) [[ $value =~ ^[0-9]+$ ]] && ((10#$value >= 1 && 10#$value <= 8)) ;; KEY_TTL_HOURS) [[ $value =~ ^[0-9]+$ ]] && ((10#$value >= 1 && 10#$value <= 720)) ;; TARGET_PROJECT_UUID) [[ -z $value ]] || is_uuid "$value" ;; *) return 1 ;; esac } plan_array_allowed() { case $1 in VOLS|BINDS|FQDNS|DMOUNTS|RUNNING|RES|DPROJECTS|PROJECT_MAPS|MSERVERS|MRES|MVOLS|MBINDS|MIMGS|MRUNNING) return 0 ;; *) return 1 ;; esac; } plan_append_array() { case $1 in VOLS) VOLS+=("$2") ;; BINDS) BINDS+=("$2") ;; FQDNS) FQDNS+=("$2") ;; DMOUNTS) DMOUNTS+=("$2") ;; RUNNING) RUNNING+=("$2") ;; RES) RES+=("$2") ;; DPROJECTS) DPROJECTS+=("$2") ;; PROJECT_MAPS) PROJECT_MAPS+=("$2") ;; MSERVERS) MSERVERS+=("$2") ;; MRES) MRES+=("$2") ;; MVOLS) MVOLS+=("$2") ;; MBINDS) MBINDS+=("$2") ;; MIMGS) MIMGS+=("$2") ;; MRUNNING) MRUNNING+=("$2") ;; *) return 1 ;; esac } plan_reset_arrays() { VOLS=(); BINDS=(); FQDNS=(); DMOUNTS=(); RUNNING=(); RES=(); DPROJECTS=(); PROJECT_MAPS=() MSERVERS=(); MRES=(); MVOLS=(); MBINDS=(); MIMGS=(); MRUNNING=() } plan_emit_array() { local key=$1 value; shift; for value in "$@"; do printf 'A\t%s\t%s\n' "$key" "$(plan_b64 "$value")"; done; } legacy_unquote() { local s=$1 needle="'\\''" [[ $s == \'*\' && ${#s} -ge 2 ]] || return 1 s=${s:1:${#s}-2}; s=${s//"$needle"/\'}; printf '%s' "$s" } load_legacy_plan() { local f=$1 line key raw value plan_reset_arrays while IFS= read -r line || [[ -n $line ]]; do if [[ $line =~ ^([A-Z_][A-Z0-9_]*)=\(\)$ ]]; then plan_array_allowed "${BASH_REMATCH[1]}" || return 1 elif [[ $line =~ ^([A-Z_][A-Z0-9_]*)\+=((\'.*\'))$ ]]; then key=${BASH_REMATCH[1]}; raw=${BASH_REMATCH[2]}; plan_array_allowed "$key" || return 1 value=$(legacy_unquote "$raw") || return 1; plan_append_array "$key" "$value" || return 1 elif [[ $line =~ ^([A-Z_][A-Z0-9_]*)=((\'.*\'))$ ]]; then key=${BASH_REMATCH[1]}; raw=${BASH_REMATCH[2]}; plan_scalar_allowed "$key" || return 1 value=$(legacy_unquote "$raw") || return 1; plan_scalar_valid "$key" "$value" || return 1; printf -v "$key" '%s' "$value" elif [[ -n $line ]]; then return 1 fi done <"$f" } load_plan_file() { local f=$1 header kind key encoded extra value IFS= read -r header <"$f" || return 1 if [[ $header != CM_PLAN_V2 ]]; then load_legacy_plan "$f"; return; fi plan_reset_arrays while IFS=$'\t' read -r kind key encoded extra; do [[ -z $extra && $kind =~ ^[SA]$ && $key =~ ^[A-Z_][A-Z0-9_]*$ && $encoded =~ ^[A-Za-z0-9+/=]*$ ]] || return 1 value=$(plan_unb64 "$encoded") || return 1 if [[ $kind == S ]]; then plan_scalar_allowed "$key" || return 1; plan_scalar_valid "$key" "$value" || return 1; printf -v "$key" '%s' "$value" else plan_array_allowed "$key" || return 1; plan_append_array "$key" "$value" || return 1; fi done < <(tail -n +2 "$f") } save_plan() { local f v; f="$(state_path).plan"; PLAN_FILE=$f; mkdir -p "$STATE_DIR/state" ( umask 077 { echo CM_PLAN_V2 for v in ENGINE DST_HOST DST_USER DST_PORT DST_AUTH DST_KEY DST_HOST_KEY SRC_LOCAL SRC_HOST SRC_USER SRC_PORT SRC_AUTH SRC_KEY SRC_HOST_KEY \ S_IP D_IP S_VERSION S_DBU S_DBN S_ARCH D_ARCH SEL_UUIDS OPT_VOLUMES OPT_BINDS OPT_IMAGES OPT_START OPT_IPREWRITE OPT_COMPRESS \ OPT_BRIDGE MODE DST_REACH S_LOCAL_PUB S_LOCAL_USER S_DOCKERCFG INSTALL_SHA256 FREEZE_TIMEOUT TRANSFER_JOBS CONSOLIDATE \ KEY_TTL_HOURS TARGET_PROJECT_UUID; do printf 'S\t%s\t%s\n' "$v" "$(plan_b64 "${!v}")" done plan_emit_array VOLS "${VOLS[@]}"; plan_emit_array BINDS "${BINDS[@]}"; plan_emit_array FQDNS "${FQDNS[@]}" plan_emit_array DMOUNTS "${DMOUNTS[@]}"; plan_emit_array RUNNING "${RUNNING[@]}"; plan_emit_array RES "${RES[@]}" plan_emit_array DPROJECTS "${DPROJECTS[@]}" plan_emit_array PROJECT_MAPS "${PROJECT_MAPS[@]}" plan_emit_array MSERVERS "${MSERVERS[@]}"; plan_emit_array MRES "${MRES[@]}"; plan_emit_array MVOLS "${MVOLS[@]}" plan_emit_array MBINDS "${MBINDS[@]}"; plan_emit_array MIMGS "${MIMGS[@]}"; plan_emit_array MRUNNING "${MRUNNING[@]}" } >"$f" ) } load_plan() { local f legacy f="$(state_path).plan" if [[ ! -f $f ]]; then legacy="$(legacy_state_path).plan"; [[ -f $legacy ]] || return 1; f=$legacy; fi load_plan_file "$f" || return 1 PLAN_FILE=$f } load_only_plan() { local plans=() p for p in "$STATE_DIR"/state/*.plan; do [[ -f $p ]] && plans+=("$p"); done ((${#plans[@]})) || die "No copy in progress found. Run the copy first." ((${#plans[@]} == 1)) || die "Multiple copies are in progress. Specify the destination with --dst user@host[:port]." load_plan_file "${plans[0]}" || die "Saved migration plan is invalid or unsafe." PLAN_FILE=${plans[0]} } validate_loaded_plan() { if ((CUTOVER)) && [[ $ENGINE == import && $SEL_UUIDS == ALL && ${#RES[@]} == 0 ]]; then die "This saved plan is missing its resource list and cannot be cut over safely. Run the copy phase again." fi } # ------------------------------------------------------------------ usage --- usage() { cat <= 1 && 10#$2 <= 8)) || die "Jobs must be 1-8"; TRANSFER_JOBS=$((10#$2)); shift ;; --freeze-timeout) [[ ${2:-} =~ ^[0-9]+$ ]] && ((10#$2 >= 5 && 10#$2 <= 900)) || die "Freeze timeout must be 5-900 seconds"; FREEZE_TIMEOUT=$((10#$2)); shift ;; --key-ttl-hours) [[ ${2:-} =~ ^[0-9]+$ ]] && ((10#$2 >= 1 && 10#$2 <= 720)) || die "Key lifetime must be 1-720 hours"; KEY_TTL_HOURS=$((10#$2)); shift ;; --installer-sha256) [[ ${2:-} =~ ^[A-Fa-f0-9]{64}$ ]] || die "Installer SHA-256 must be 64 hexadecimal characters"; INSTALL_SHA256=$(tr A-F a-f <<<"$2"); shift ;; --no-auto-rollback) AUTO_ROLLBACK=0 ;; --relay) FORCE_RELAY=1 ;; --coolify-version) is_version "$2" || die "Invalid version"; COOLIFY_VERSION_OVERRIDE=$2; shift ;; --detach) DETACH=1 ;; --no-detach) DETACH=0 ;; --attach) ATTACH=1 ;; --plan-only) PLAN_ONLY=1 ;; -y|--yes) ASSUME_YES=1 ;; -h|--help) usage; exit 0 ;; *) usage; die "Unknown option: $(clean "$1")" ;; esac shift done for arg in "$SRC_HOST_KEY" "$DST_HOST_KEY"; do [[ -z $arg || $arg =~ ^SHA256:[A-Za-z0-9+/=]+$ ]] || die "Host-key fingerprints must start with SHA256:"; done [[ $KEY_TTL_HOURS =~ ^[0-9]+$ ]] && ((10#$KEY_TTL_HOURS >= 1 && 10#$KEY_TTL_HOURS <= 720)) || die "Key lifetime must be 1-720 hours" if [[ -n $SEL_UUIDS ]]; then local u; for u in ${SEL_UUIDS//,/ }; do is_uuid "$u" || die "Invalid resource uuid: $(clean "$u")"; done; fi ((CLEANUP_ORPHANS && (UNDO || CUTOVER))) && die "--cleanup-orphans cannot be combined with --undo or --cutover" if ((ASSUME_YES)) || ! (: <"$TTY") 2>/dev/null; then INTERACTIVE=0; fi } # -------------------------------------------------------------- discovery --- # Strict parser: anything that doesn't validate is dropped (with a log line), never used. parse_discovery() { # role (reads stdin) local r=$1 line k v f while IFS= read -r line; do k=${line%%=*} v=${line#*=} case $r:$k in SRC:FATAL|DST:FATAL) die "$r: $(clean "$v")" ;; SRC:VERSION) is_version "$v" && S_VERSION=$v ;; SRC:ARCH) [[ $v =~ ^[a-z0-9_]{1,16}$ ]] && S_ARCH=$v ;; SRC:OS) S_OS=$(clean "$v") ;; SRC:IP) is_ip4 "$v" && S_IP=$v ;; SRC:MID) [[ $v =~ ^[a-f0-9]{0,64}$ ]] && S_MID=$v ;; SRC:DATA) S_DATA=$(num "$v") ;; SRC:DBU) is_name "$v" && S_DBU=$v ;; SRC:DBN) is_name "$v" && S_DBN=$v ;; SRC:COUNTS) S_COUNTS=$(clean "$v") ;; SRC:LOCALPUB) [[ $v =~ ^(ssh-ed25519|ssh-rsa|ecdsa-sha2-nistp[0-9]+)\ [A-Za-z0-9+/=]+( [^\'\"]{0,200})?$ ]] && S_LOCAL_PUB=$v ;; SRC:LOCALIP) S_LOCAL_IP=$(clean "$v") ;; SRC:LOCALUSER) is_user "$v" && S_LOCAL_USER=$v ;; SRC:NREMOTE) N_REMOTE=$(num "$v") ;; SRC:MWARN) warn "Managed server: $(clean "$v")" ;; SRC:DOCKERCFG) is_abspath "$v" && [[ $v == */.docker/config.json ]] && S_DOCKERCFG=$v ;; SRC:FQDN) is_fqdn "$v" && FQDNS+=("$v") ;; SRC:RES) IFS=$'\x1f' read -r -a f <<<"$v" [[ ${f[0]} =~ ^(application|service|database)$ ]] && is_uuid "${f[1]}" && is_uuid "${f[5]:-}" && RES+=("${f[0]}$US${f[1]}$US$(clean "${f[2]}")$US$(clean "${f[3]}")$US$(clean "${f[4]}")$US${f[5]}") ;; SRC:VOL) IFS='|' read -r -a f <<<"$v" if is_name "${f[0]}" && is_abspath "${f[3]}" && [[ ${f[4]} =~ ^[A-Za-z0-9+/=]*$ ]]; then VOLS+=("${f[0]}|$(num "${f[1]}")|$([[ ${f[2]} == 1 ]] && echo 1)|${f[3]}|${f[4]}|$(clean "${f[5]}" | tr -cd 'A-Za-z0-9_.,-')") else log "dropped invalid volume record"; fi ;; SRC:BIND) IFS='|' read -r -a f <<<"$v" if bind_ok "${f[0]}"; then BINDS+=("${f[0]}|$(num "${f[1]}")|$(clean "${f[2]}" | tr -cd 'A-Za-z0-9_.-')") else case ${f[0]} in /var/run/docker.sock|/run/docker.sock|/data/coolify/*) ;; *) [[ " ${SKIPPED_BINDS[*]} " == *" $(clean "${f[0]}") "* ]] || SKIPPED_BINDS+=("$(clean "${f[0]}")") ;; esac; fi ;; SRC:IMG) IFS='|' read -r -a f <<<"$v" is_image "${f[0]}" && IMGS+=("${f[0]}|$(num "${f[1]}")") ;; SRC:RUN) IFS='|' read -r -a f <<<"$v" is_name "${f[0]}" && is_image "${f[1]}" && RUNNING+=("${f[0]}|${f[1]}") ;; SRC:MSERVER) IFS=$'\x1f' read -r -a f <<<"$v" if [[ ${f[0]} =~ ^[0-9]+$ ]] && is_name "${f[1]}" && [[ ${f[2]} =~ ^[A-Za-z0-9.:-]{1,253}$ ]] && is_port "${f[3]}" && is_user "${f[4]}" && is_uuid "${f[5]}" && [[ ${f[6]} =~ ^[a-z0-9_]{1,16}$ ]]; then MSERVERS+=("${f[0]}$US${f[1]}$US${f[2]}$US${f[3]}$US${f[4]}$US${f[5]}$US${f[6]}") else log "dropped invalid managed-server record"; fi ;; SRC:MRES) IFS=$'\x1f' read -r -a f <<<"$v" if [[ ${f[0]} =~ ^(application|service|database)$ ]] && is_uuid "${f[1]}" && [[ ${f[5]} =~ ^[0-9]+$ ]] && is_name "${f[6]}"; then MRES+=("${f[0]}$US${f[1]}$US$(clean "${f[2]}")$US$(clean "${f[3]}")$US$(clean "${f[4]}")$US${f[5]}$US${f[6]}") else log "dropped invalid managed-resource record"; fi ;; SRC:MVOL) IFS=$'\x1f' read -r -a f <<<"$v" if [[ ${f[0]} =~ ^[0-9]+$ ]] && is_name "${f[1]}" && is_name "${f[2]}" && is_abspath "${f[5]}" && [[ ${f[6]} =~ ^[A-Za-z0-9+/=]*$ ]]; then MVOLS+=("${f[0]}$US${f[1]}$US${f[2]}$US$(num "${f[3]}")$US$([[ ${f[4]} == 1 ]] && echo 1)$US${f[5]}$US${f[6]}$US$(clean "${f[7]}" | tr -cd 'A-Za-z0-9_.,-')") else log "dropped invalid managed-volume record"; fi ;; SRC:MBIND) IFS=$'\x1f' read -r -a f <<<"$v" if [[ ${f[0]} =~ ^[0-9]+$ ]] && is_name "${f[1]}" && bind_ok "${f[2]}"; then MBINDS+=("${f[0]}$US${f[1]}$US${f[2]}$US$(num "${f[3]}")$US$(clean "${f[4]}" | tr -cd 'A-Za-z0-9_.-')") else log "dropped invalid managed-bind record"; fi ;; SRC:MIMG) IFS=$'\x1f' read -r -a f <<<"$v" [[ ${f[0]} =~ ^[0-9]+$ ]] && is_name "${f[1]}" && is_image "${f[2]}" && MIMGS+=("${f[0]}$US${f[1]}$US${f[2]}$US$(num "${f[3]}")") ;; SRC:MRUN) IFS=$'\x1f' read -r -a f <<<"$v" [[ ${f[0]} =~ ^[0-9]+$ ]] && is_name "${f[1]}" && is_name "${f[2]}" && is_image "${f[3]}" && MRUNNING+=("${f[0]}$US${f[1]}$US${f[2]}$US${f[3]}") ;; DST:ARCH) [[ $v =~ ^[a-z0-9_]{1,16}$ ]] && D_ARCH=$v ;; DST:OS) D_OS=$(clean "$v") ;; DST:IP) is_ip4 "$v" && D_IP=$v ;; DST:MID) [[ $v =~ ^[a-f0-9]{0,64}$ ]] && D_MID=$v ;; DST:FREE) D_FREE=$(num "$v") ;; DST:INODES) D_INODES=$(num "$v") ;; DST:CPU) D_CPU=$(num "$v") ;; DST:MEMTOTAL) D_MEM_TOTAL=$(num "$v") ;; DST:DPROJECT) IFS=$'\x1f' read -r -a f <<<"$v" is_uuid "${f[0]}" && [[ -n ${f[1]:-} ]] && DPROJECTS+=("${f[0]}$US$(clean "${f[1]}")") ;; DST:HASDOCKER) D_HASDOCKER=1 ;; DST:COOLIFY) is_version "$v" && D_COOLIFY=$v ;; DST:COOLIFYUSERS) D_USERS=$(num "$v") ;; DST:DATA) D_DATA=1 ;; DST:JOURNAL) D_JOURNAL=$(num "$v") ;; esac done } discover() { local out ui_capture out "Inspecting the old Coolify and its workloads" rscript SRC "$RS_DISCOVER_SRC" || die "Discovery on the old server failed" parse_discovery SRC <<<"$out" if ((N_REMOTE > 0)); then ui_capture out "Inspecting $N_REMOTE managed server(s)" rscript SRC "$RS_DISCOVER_MANAGED" || die "Managed-server discovery on the old Coolify failed" parse_discovery SRC <<<"$out" fi ui_capture out "Checking destination capacity and projects" rscript DST "$RS_DISCOVER_DST" || die "Discovery on the new server failed" parse_discovery DST <<<"$out" [[ -n $S_MID && $S_MID == "$D_MID" ]] && die "Old and new server are the same machine." [[ -n $S_IP && $S_IP == "$D_IP" && -n $S_MID && $S_MID == "$D_MID" ]] && die "Old and new server are the same machine." [[ -n $COOLIFY_VERSION_OVERRIDE ]] && S_VERSION=$COOLIFY_VERSION_OVERRIDE [[ -z $D_IP && $DST_HOST =~ ^[0-9.]+$ ]] && is_ip4 "$DST_HOST" && D_IP=$DST_HOST [[ -n $D_IP ]] || warn "Couldn't determine the new server's public IP — IP rewrite and bridge will be skipped." ((${#SKIPPED_BINDS[@]})) && warn "Not copying ${#SKIPPED_BINDS[@]} bind mount(s) in system locations: ${SKIPPED_BINDS[*]}" } sum_field() { local t=0 x f v; f=$1; shift; for x in "$@"; do IFS='|' read -r -a p <<<"$x"; v=$(num "${p[$f]}"); t=$((t + v)); done; echo $t; } sum_us_field() { local t=0 x f v p; f=$1; shift; for x in "$@"; do IFS=$'\x1f' read -r -a p <<<"$x"; v=$(num "${p[$f]}"); t=$((t + v)); done; echo $t; } # ------------------------------------------------------------------- plan --- res_label() { local f; IFS=$'\x1f' read -r -a f <<<"$1"; printf '%-11s %s %s(%s / %s)%s' "${f[0]}" "${f[2]}" "$D" "${f[3]}" "${f[4]}" "$R"; } choose_what() { local idx sel i u n if [[ -n $D_COOLIFY && ${D_USERS:-0} -gt 0 ]]; then TARGET_KIND=existing info "The new server already runs ${B}Coolify $D_COOLIFY${R}. Resources will be ${B}added${R} to it — nothing there is changed or deleted." elif [[ -n $D_COOLIFY || -n $D_DATA ]]; then [[ ${D_JOURNAL:-0} -gt 0 ]] && info "Found a previous unfinished copy on the new server — it will be reused." TARGET_KIND=empty [[ ${D_JOURNAL:-0} -gt 0 ]] || { warn "The new server has a Coolify install with no users or leftover /data/coolify." ui_confirm "Treat it as empty and overwrite it?" n || die "Aborted — use --undo or clean the server first."; OVERWRITE=1; } else TARGET_KIND=empty fi if [[ -z $WHAT ]]; then if ((INTERACTIVE)); then ui_select idx "What do you want to copy?" \ "Everything ${D}(the whole Coolify: settings, users, all apps, databases, services)${R}" \ "Pick apps / databases / services ${D}(${#RES[@]} on this server)${R}" ((idx == 0)) && WHAT=all || WHAT=selected else WHAT=all; fi fi if [[ $WHAT == selected && -z $SEL_UUIDS ]]; then ((${#RES[@]})) || die "No apps, databases or services found on the old server." local labels=() defs="" for i in "${RES[@]}"; do labels+=("$(res_label "$i")"); defs+="0 "; done ui_multi sel "Which ones?" "$defs" "${labels[@]}" n=0; SEL_UUIDS="" for i in $sel; do [[ $i == 1 ]] && { IFS=$'\x1f' read -r -a u <<<"${RES[n]}"; SEL_UUIDS+="${u[1]},"; }; n=$((n + 1)); done SEL_UUIDS=${SEL_UUIDS%,} [[ -n $SEL_UUIDS ]] || die "Nothing selected." fi if ((N_REMOTE > 0)) && [[ $WHAT == all && $TARGET_KIND == empty ]] && ((INTERACTIVE)) && ((CONSOLIDATE == 0)); then ui_select idx "The old Coolify manages $N_REMOTE additional server(s). What should happen to their workloads?" \ "Keep them on their current servers ${D}(clone the Coolify control plane only)${R}" \ "Move them onto the new server ${D}(consolidate everything into one localhost)${R}" ((idx == 1)) && CONSOLIDATE=1 fi if ((CONSOLIDATE)); then [[ $WHAT == all && $TARGET_KIND == empty ]] || die "--consolidate requires --all and an empty destination server." validate_consolidation fi if [[ $WHAT == all && $TARGET_KIND == empty ]]; then ENGINE=clone; else ENGINE=import; fi [[ $WHAT == all && $TARGET_KIND == existing ]] && SEL_UUIDS=ALL log "engine=$ENGINE target=$TARGET_KIND what=$WHAT sel=$SEL_UUIDS" } project_map_payload() { local item f { for item in "${PROJECT_MAPS[@]}"; do IFS=$'\x1f' read -r -a f <<<"$item" is_uuid "${f[0]:-}" && is_uuid "${f[2]:-}" && printf '%s\x1f%s\n' "${f[0]}" "${f[2]}" done } | base64 | tr -d '\r\n' } choose_target_project() { local idx=0 item sf df found=0 seen="," labels=() [[ $TARGET_KIND == existing && $ENGINE == import && ${#DPROJECTS[@]} -gt 0 ]] || return 0 for item in "${DPROJECTS[@]}"; do IFS=$'\x1f' read -r -a df <<<"$item" [[ ${df[0]} == "$TARGET_PROJECT_UUID" ]] && found=1 done if [[ -n $TARGET_PROJECT_UUID ]]; then ((found)) || die "Destination project $TARGET_PROJECT_UUID was not found in the root team." return 0 fi ((INTERACTIVE)) || return 0 PROJECT_MAPS=() for item in "${RES[@]}"; do IFS=$'\x1f' read -r -a sf <<<"$item" [[ $SEL_UUIDS == ALL || ",$SEL_UUIDS," == *",${sf[1]},"* ]] || continue is_uuid "${sf[5]:-}" || die "Source project identity is missing for ${sf[2]}. Run the latest migration script again." [[ $seen == *",${sf[5]},"* ]] && continue seen+="${sf[5]}," labels=("Keep / create ${sf[3]} on the destination ${D}(same project name)${R}") for item in "${DPROJECTS[@]}"; do IFS=$'\x1f' read -r -a df <<<"$item" labels+=("${df[1]} ${D}(${df[0]})${R}") done ui_select idx "Where should source project ${B}${sf[3]}${R} be placed?" "${labels[@]}" if ((idx > 0)); then IFS=$'\x1f' read -r -a df <<<"${DPROJECTS[idx - 1]}" PROJECT_MAPS+=("${sf[5]}$US${sf[3]}$US${df[0]}$US${df[1]}") ok "${B}${sf[3]}${R} ${G_ARR} ${B}${df[1]}${R} ${D}(environment names are preserved)${R}" else ok "${B}${sf[3]}${R} ${G_ARR} same project name on the destination" fi done } validate_consolidation() { local r f sid name v other of n owners="" seen="" found ((${#MRES[@]})) || { warn "No workloads are assigned to managed servers; consolidation has nothing extra to move."; return 0; } for r in "${MRES[@]}"; do IFS=$'\x1f' read -r -a f <<<"$r"; sid=${f[5]}; name=${f[6]}; found=0 for v in "${MSERVERS[@]}"; do IFS=$'\x1f' read -r -a of <<<"$v"; [[ ${of[0]} == "$sid" ]] && found=1; done ((found)) || die "Managed server $name could not be reached. Consolidation will not start until every workload source is reachable with passwordless sudo." done # Docker volume names and bind destinations cannot represent two different # source directories on one host. Abort instead of silently mixing data. for r in "${MVOLS[@]}"; do IFS=$'\x1f' read -r -a f <<<"$r"; n=${f[2]}; owners="${f[1]}" for v in "${VOLS[@]}"; do [[ ${v%%|*} == "$n" ]] && die "Volume name collision: $n exists on localhost and ${f[1]}. Rename one before consolidating."; done for other in "${MVOLS[@]}"; do [[ $other == "$r" ]] && continue; IFS=$'\x1f' read -r -a of <<<"$other" [[ ${of[2]} == "$n" && ${of[0]} != "${f[0]}" ]] && die "Volume name collision: $n exists on ${f[1]} and ${of[1]}. Rename one before consolidating." done done for r in "${MBINDS[@]}"; do IFS=$'\x1f' read -r -a f <<<"$r"; n=${f[2]} for v in "${BINDS[@]}"; do other=${v%%|*} [[ $other == "$n" || $other == "$n"/* || $n == "$other"/* ]] && die "Bind-path collision: $n on ${f[1]} overlaps $other on localhost. Use non-overlapping paths before consolidating." done for other in "${MBINDS[@]}"; do [[ $other == "$r" ]] && continue; IFS=$'\x1f' read -r -a of <<<"$other" [[ ${of[0]} != "${f[0]}" ]] && [[ ${of[2]} == "$n" || ${of[2]} == "$n"/* || $n == "${of[2]}"/* ]] && die "Bind-path collision: $n on ${f[1]} overlaps ${of[2]} on ${of[1]}. Use non-overlapping paths before consolidating." done done ok "Consolidation preflight: ${#MRES[@]} managed workload(s) from ${#MSERVERS[@]} server(s) can be mapped to the new localhost" } # Narrow volumes/binds/images/data dirs to the selected resources (import engine) uuid_selected() { [[ $SEL_UUIDS == ALL ]] && return 0; local u; for u in ${SEL_UUIDS//,/ }; do [[ $1 == *"$u"* ]] && return 0; done; return 1; } narrow_to_selection() { [[ $ENGINE == import ]] || return 0 local v keep=() p for v in "${VOLS[@]}"; do IFS='|' read -r -a p <<<"$v"; uuid_selected "${p[0]},${p[5]}" && keep+=("$v"); done; VOLS=("${keep[@]}") keep=(); for v in "${BINDS[@]}"; do IFS='|' read -r -a p <<<"$v"; uuid_selected "${p[2]}" && keep+=("$v"); done; BINDS=("${keep[@]}") keep=(); for v in "${IMGS[@]}"; do uuid_selected "${v%%|*}" && keep+=("$v"); done; IMGS=("${keep[@]}") keep=(); for v in "${RUNNING[@]}"; do uuid_selected "${v%%|*}" && keep+=("$v"); done; RUNNING=("${keep[@]}") } choose_options() { local sel s f CROSS_ARCH=0 [[ -n $S_ARCH && -n $D_ARCH && $S_ARCH != "$D_ARCH" ]] && { CROSS_ARCH=1 warn "CPU architecture differs ($S_ARCH ${G_ARR} $D_ARCH): old images cannot run on the new CPU. Apps will be built for $D_ARCH from each last successful commit while the old server stays live." OPT_IMAGES=0 } if ((CONSOLIDATE)); then for s in "${MSERVERS[@]}"; do IFS=$'\x1f' read -r -a f <<<"$s" if [[ -n ${f[6]:-} && ${f[6]} != "$D_ARCH" ]]; then CROSS_ARCH=1 OPT_IMAGES=0 warn "Managed server ${f[1]} uses ${f[6]} while the destination uses $D_ARCH. All apps will rebuild from their last successful commits so restart behavior is consistent." break fi done fi ((${#IMGS[@]} == 0)) && OPT_IMAGES=0 [[ $ENGINE == import ]] && OPT_REMOTES=0 ((INTERACTIVE)) || return 0 echo ui_multi sel "Options" \ "$OPT_VOLUMES $OPT_BINDS $OPT_IMAGES $OPT_START $OPT_PAUSE_TASKS $OPT_IPREWRITE $OPT_COMPRESS" \ "Copy volume data ${D}($((${#VOLS[@]} + ${#MVOLS[@]} * CONSOLIDATE)) volumes; databases are frozen a few seconds for a consistent copy)${R}" \ "Copy bind-mounted folders ${D}($((${#BINDS[@]} + ${#MBINDS[@]} * CONSOLIDATE)) folders)${R}" \ "Copy built images ${D}($((${#IMGS[@]} + ${#MIMGS[@]} * CONSOLIDATE)) images, skips rebuilds)${R}" \ "Start them on the new server so you can test ${D}(runs in parallel with the old one)${R}" \ "Pause scheduled tasks & backups on the new server until cutover ${D}(no double cron jobs/backups)${R}" \ "Rewrite old IP ${G_ARR} new IP ${D}(sslip domains, env vars, labels)${R}" \ "Compress in transit ${D}(turn off on fast LAN links)${R}" read -r OPT_VOLUMES OPT_BINDS OPT_IMAGES OPT_START OPT_PAUSE_TASKS OPT_IPREWRITE OPT_COMPRESS <<<"$sel" } running_spec() { local r p out="" for r in "${RUNNING[@]}"; do out+="${out:+,}$r"; done if ((CONSOLIDATE)); then for r in "${MRUNNING[@]}"; do IFS=$'\x1f' read -r -a p <<<"$r"; out+="${out:+,}${p[2]}|${p[3]}"; done fi printf '%s' "$out" } estimated_bytes() { local total=0 [[ $ENGINE == clone ]] && total=$S_DATA ((OPT_VOLUMES)) && total=$((total + $(sum_field 1 "${VOLS[@]}"))) ((OPT_BINDS)) && total=$((total + $(sum_field 1 "${BINDS[@]}"))) ((OPT_IMAGES)) && total=$((total + $(sum_field 1 "${IMGS[@]}"))) if ((CONSOLIDATE)); then ((OPT_VOLUMES)) && total=$((total + $(sum_us_field 3 "${MVOLS[@]}"))) ((OPT_BINDS)) && total=$((total + $(sum_us_field 3 "${MBINDS[@]}"))) ((OPT_IMAGES)) && total=$((total + $(sum_us_field 3 "${MIMGS[@]}"))) fi printf '%s' "$total" } check_destination_capacity() { # estimated incoming bytes local incoming=$1 reserve=$((2 * 1024 * 1024 * 1024)) minimum=$((10 * 1024 * 1024 * 1024)) required ((incoming / 10 > reserve)) && reserve=$((incoming / 10)) required=$((incoming + reserve)) ((required < minimum)) && required=$minimum if ((D_CPU < 2)); then err "Destination has $D_CPU CPU core(s); Coolify requires at least 2 before migration."; return 1; fi if ((D_MEM_TOTAL < 2 * 1024 * 1024 * 1024)); then err "Destination has $(hsize "$D_MEM_TOTAL") RAM; Coolify requires at least 2 GB before migration."; return 1; fi if ((D_FREE < required)); then err "Destination has not enough free disk: $(hsize "$D_FREE") available, $(hsize "$required") required (Coolify minimum or incoming data plus safety reserve, whichever is larger)." return 1 fi if ((D_INODES < 100000)); then err "Destination has only $D_INODES free inodes; at least 100000 are required before migration."; return 1; fi ok "Destination capacity: ${D_CPU} CPU, $(hsize "$D_MEM_TOTAL") RAM, $(hsize "$D_FREE") free disk, $D_INODES free inodes" } show_plan() { local total what p f sep=""; total=$(estimated_bytes) yn() { (($1)) && printf '%s%s%s' "$GRN" "$G_OK" "$R" || printf '%s%s%s' "$D" "$G_NDASH" "$R"; } if [[ $ENGINE == clone ]]; then if ((CONSOLIDATE)); then what="everything from localhost + ${#MSERVERS[@]} managed server(s), consolidated onto the new localhost" else what="everything (full Coolify clone onto an empty server)"; fi elif [[ $SEL_UUIDS == ALL ]]; then what="all apps/databases/services, added into the existing Coolify" else what="$(tr -cd ',' <<<"$SEL_UUIDS" | wc -c | awk '{print $1+1}') selected resource(s)$([[ $TARGET_KIND == existing ]] && echo ', added into the existing Coolify' || echo ' onto a fresh Coolify')"; fi echo; hr printf ' %sOLD%s %s %s(%s %s %s %s %s)%s\n' "$B" "$R" "$(target SRC)" "$D" "$S_OS" "$G_DOT" "$S_ARCH" "$G_DOT" "${S_IP:-?}" "$R" printf ' %sNEW%s %s %s(%s %s %s %s %s)%s\n' "$B" "$R" "$(target DST)" "$D" "$D_OS" "$G_DOT" "$D_ARCH" "$G_DOT" "${D_IP:-?}" "$R" printf ' %sCopy%s %s\n' "$B" "$R" "$what" if ((${#PROJECT_MAPS[@]})); then printf ' %sMap%s ' "$B" "$R" for p in "${PROJECT_MAPS[@]}"; do IFS=$'\x1f' read -r -a f <<<"$p" printf '%s%s %s %s' "$sep" "${f[1]}" "$G_ARR" "${f[3]}" sep=" $G_DOT " done printf '\n' fi printf ' %sSize%s ~%s %s(%s, %s free on the new server)%s\n' "$B" "$R" "$(hsize $total)" "$D" "$MODE" "$(hsize "$D_FREE")" "$R" hr printf ' %s old server keeps running the whole time %s nothing there is stopped until you run --cutover\n' "$(yn 1)" "$G_DASH" printf ' %s volumes (%d) %s binds (%d) %s images (%d) %s start on new %s pause tasks %s IP rewrite\n' \ "$(yn $OPT_VOLUMES)" $((${#VOLS[@]} + ${#MVOLS[@]} * CONSOLIDATE)) \ "$(yn $OPT_BINDS)" $((${#BINDS[@]} + ${#MBINDS[@]} * CONSOLIDATE)) \ "$(yn $OPT_IMAGES)" $((${#IMGS[@]} + ${#MIMGS[@]} * CONSOLIDATE)) "$(yn $OPT_START)" "$(yn $OPT_PAUSE_TASKS)" "$(yn $OPT_IPREWRITE)" printf ' %s automatic rollback if anything fails %s\n' "$(yn $AUTO_ROLLBACK)" \ "$([[ $TARGET_KIND == existing ]] && echo "(removes only what was added)" || echo "(new server is wiped back to empty)")" ((${#BINDS[@]})) && { info "Bind folders to copy:"; local b; for b in "${BINDS[@]}"; do printf ' %s\n' "${b%%|*}"; done; } hr ((OPT_START)) && warn "Started copies run in parallel with the old ones: background workers/cron jobs ${B}inside${R} your apps would run twice until cutover." } # --------------------------------------------------------------- transfer --- setup_transfer() { ui_run "Preparing old server (rsync, zstd)" rscript SRC "$RS_PREPARE" || die "Preparing the old server failed" ui_run "Preparing new server (rsync, zstd)" rscript DST "$RS_PREPARE" || die "Preparing the new server failed" if rscript SRC 'command -v zstd' >/dev/null 2>&1 && rscript DST 'command -v zstd' >/dev/null 2>&1; then COMP="zstd -T0 -3 -c" DECOMP="zstd -dc" fi ((FORCE_RELAY && CONSOLIDATE)) && die "--relay cannot safely consolidate live databases. Each managed source server must reach the new server directly for the bounded incremental freeze." ((FORCE_RELAY)) && { MODE=relay; warn "Relay mode: data streams through this machine"; return; } local pub direct reach=${DST_REACH:-$DST_HOST} rport=$DST_PORT hostkeys if [[ $reach == *:* && $reach != *:*:* ]]; then rport=${reach##*:}; reach=${reach%%:*}; fi [[ $reach =~ ^[A-Za-z0-9.-]{1,253}$ ]] && is_port "$rport" || die "Invalid --dst-reach" TRANSFER_REACH=$reach TRANSFER_PORT=$rport pub=$(rscript SRC "$RS_KEYGEN" 2>>"$LOG" | tail -n 1) [[ $pub =~ ^ssh-ed25519\ [A-Za-z0-9+/=]+\ $KEY_MARKER$ ]] || { warn "Could not create a transfer key on the old server ${G_ARR} relay mode"; MODE=relay; return; } # Restricted and time-limited even if this run is killed before cleanup. local exp; exp=$(expiry_utc_hours "$KEY_TTL_HOURS") rscript DST "$RS_AUTHORIZE" PUB="$pub" TUSER="$DST_USER" OPTS="restrict,expiry-time=\"$exp\"" >>"$LOG" 2>&1 || { MODE=relay; return; } TRUST_SET=1 # Pin the new server's host keys (fetched over our already-authenticated connection) — no blind trust. hostkeys=$(rscript DST 'cat /etc/ssh/ssh_host_*_key.pub 2>/dev/null' 2>>"$LOG" | awk 'NF>=2 && $1 ~ /^(ssh-|ecdsa-)/ {print $1" "$2}') while :; do local khspec=$reach; [[ $rport != 22 ]] && khspec="[$reach]:$rport" rscript SRC "umask 077; : >\"\$RWD/known_hosts\"; printf '%s\n' \"\$KEYS\" | while read -r t k; do [ -n \"\$k\" ] && printf '%s %s %s\n' \"\$SPEC\" \"\$t\" \"\$k\" >>\"\$RWD/known_hosts\"; done" \ KEYS="$hostkeys" SPEC="$khspec" >>"$LOG" 2>&1 DSSH="ssh -i $RWD/id_ed25519 -p $rport -o BatchMode=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile=$RWD/known_hosts -o ServerAliveInterval=15 -o LogLevel=ERROR" DT="$DST_USER@$reach" ui_capture direct "Testing the direct server-to-server route" rscript SRC "$RS_TEST_DIRECT" DSSH="$DSSH" DT="$DT" if [[ $direct == *DIRECT_OK* ]]; then MODE=direct; ok "Direct server-to-server link ${D}(${S_IP:-old} ${G_ARR} $reach, pinned host key, key expires in $KEY_TTL_HOURS hour(s))${R}" setup_managed_transfer "$reach" "$rport" return fi warn "The old server can't reach the new one at $reach:$rport" if ((INTERACTIVE)) && ui_confirm "Try another address (e.g. a private IP)?" n; then ui_ask reach "New server's address as seen from the old one" "$reach" [[ $reach =~ ^[A-Za-z0-9.-]{1,253}$ ]] || reach=$DST_HOST continue fi MODE=relay setup_managed_transfer "$reach" "$rport" warn "Falling back to relay mode for localhost data; managed servers still transfer directly" return done } setup_managed_transfer() { # destination address port local reach=$1 rport=$2 s p ((CONSOLIDATE)) || return 0 for s in "${MSERVERS[@]}"; do IFS=$'\x1f' read -r -a p <<<"$s" ui_run "Securing direct transfer from ${p[1]}" rscript SRC "$RS_MANAGED_SETUP" SID="${p[0]}" REACH="$reach" RPORT="$rport" \ DUSER="$DST_USER" DSUDO="$DST_SUDO" || die "${p[1]} cannot reach $reach:$rport. Allow SSH from that server to the new server, then retry." done } # Copy a list of records: label US src US dst US mode US volume transfer() { # title pairs [excludes] local title=$1 pairs=$2 excl=${3:-} label src dst mode vol [[ -z $pairs ]] && return 0 if [[ $MODE == direct ]]; then ui_stream "$title" rscript SRC "$RS_RSYNC" DSSH="$DSSH" DT="$DT" DS="$DST_SUDO" PAIRS="$pairs" ZIP="$OPT_COMPRESS" EXCLUDES="$excl" FREEZE_TIMEOUT="$FREEZE_TIMEOUT" JOBS="$TRANSFER_JOBS" else local rc=0 ex="" x set -f; for x in $excl; do ex+=" --exclude=$(sq "./${x%/}")"; done; set +f while IFS=$'\x1f' read -r label src dst mode vol; do [[ -z $label ]] && continue ui_run "$title: $label" relay_copy "$src" "$dst" "$ex" "$mode" || rc=1 done <<<"$pairs" return $rc fi } relay_copy() { # src dst excludes mode is_abspath "$1" && is_abspath "$2" || { echo "refusing unsafe path"; return 1; } case $2 in /|/etc|/usr|/var|/var/lib|/root|/home|/boot|/opt|/srv) echo "refusing unsafe destination"; return 1 ;; esac if ! rssh SRC "$SRC_SUDO test -d $(sq "$1")"; then # single file rssh SRC "$SRC_SUDO tar -C $(sq "$(dirname "$1")") -cpf - $(sq "$(basename "$1")") | $COMP" | rssh DST "$DST_SUDO mkdir -p -- $(sq "$(dirname "$2")") && $DECOMP | $DST_SUDO tar -C $(sq "$(dirname "$2")") --numeric-owner -xpf -" return fi # tar can't mirror deletions: clear the target first when mirroring (validated path above) [[ $4 == *delete* ]] && rssh DST "$DST_SUDO find $(sq "$2") -mindepth 1 -delete 2>/dev/null; true" rssh SRC "$SRC_SUDO tar -C $(sq "$1") $3 -cpf - . | $COMP" | rssh DST "$DST_SUDO mkdir -p -- $(sq "$2") && $DECOMP | $DST_SUDO tar -C $(sq "$2") --numeric-owner -xpf -" } rec() { printf '%s\x1f%s\x1f%s\x1f%s\x1f%s\n' "$@"; } pairs_data() { rec "control plane (/data/coolify)" /data/coolify /data/coolify delete "" [[ -n $S_DOCKERCFG ]] && rec "registry logins" "$S_DOCKERCFG" "$S_DOCKERCFG" "" "" true } sel_uuid_list() { # the selected uuids, expanding ALL local csv; csv=$(selected_uuid_csv) [[ -n $csv ]] && printf '%s\n' ${csv//,/ } } selected_uuid_csv() { if [[ $SEL_UUIDS != ALL ]]; then printf '%s' "$SEL_UUIDS"; return; fi local i f out="" for i in "${RES[@]}"; do IFS=$'\x1f' read -r -a f <<<"$i" is_uuid "${f[1]:-}" && out+="${out:+,}${f[1]}" done printf '%s' "$out" } pairs_resource_dirs() { # import engine: per-resource config dirs local u d for u in $(sel_uuid_list); do for d in applications databases services; do rec "files $d/$u" "/data/coolify/$d/$u" "/data/coolify/$d/$u" "" ""; done done } pairs_volumes() { # mode: "freeze" (consistent while live) or "" local v name mp m p for v in "${VOLS[@]}"; do IFS='|' read -r -a p <<<"$v"; name=${p[0]}; mp="" for m in "${DMOUNTS[@]}"; do [[ ${m%%|*} == "$name" ]] && mp=${m#*|}; done [[ -n $mp && -n ${p[3]} ]] && is_abspath "$mp" && rec "volume $name" "${p[3]}" "$mp" "delete${1:+ $1}" "$name" done } pairs_binds() { local v; for v in "${BINDS[@]}"; do rec "bind ${v%%|*}" "${v%%|*}" "${v%%|*}" "" ""; done; } create_volumes() { local spec="" v out p for v in "${VOLS[@]}"; do IFS='|' read -r -a p <<<"$v"; spec+="${p[0]}|${p[4]}"$'\n'; done if ((CONSOLIDATE)); then for v in "${MVOLS[@]}"; do IFS=$'\x1f' read -r -a p <<<"$v"; spec+="${p[2]}|${p[6]}"$'\n'; done fi ui_capture out "Preparing destination volume mountpoints" rscript DST "$RS_CREATE_VOLUMES" SPEC="$spec" || return 1 DMOUNTS=() local mp while IFS= read -r v; do [[ $v == MP=* ]] && { mp=${v#MP=}; is_name "${mp%%|*}" && is_abspath "${mp#*|}" && DMOUNTS+=("$mp"); }; done <<<"$out" ok "Prepared ${#DMOUNTS[@]} volumes on the new server" } managed_mountpoint() { # volume name local m; for m in "${DMOUNTS[@]}"; do [[ ${m%%|*} == "$1" ]] && { printf '%s' "${m#*|}"; return 0; }; done; return 1 } managed_copy() { # title server-id source destination freeze volume local title=$1 sid=$2 src=$3 dst=$4 freeze=$5 vol=$6 is_abspath "$src" && is_abspath "$dst" && [[ $sid =~ ^[0-9]+$ ]] || return 1 case $dst in /|/etc|/usr|/var|/var/lib|/var/lib/docker|/root|/home|/boot|/opt|/srv) return 1 ;; esac ui_run "$title" managed_copy_stream "$sid" "$src" "$dst" "$freeze" "$vol" } managed_copy_stream() { local sid=$1 src=$2 dst=$3 freeze=$4 vol=$5 rscript SRC "$RS_MANAGED_RSYNC" SID="$sid" SRC_PATH="$src" DST_PATH="$dst" VOLUME="$vol" FREEZE="$freeze" \ FREEZE_TIMEOUT="$FREEZE_TIMEOUT" REACH="$TRANSFER_REACH" RPORT="$TRANSFER_PORT" DUSER="$DST_USER" DSUDO="$DST_SUDO" ZIP="$OPT_COMPRESS" } transfer_managed_volumes() { # freeze 0/1 local freeze=$1 v p mp rc=0 label ((CONSOLIDATE && OPT_VOLUMES)) || return 0 for v in "${MVOLS[@]}"; do IFS=$'\x1f' read -r -a p <<<"$v"; mp=$(managed_mountpoint "${p[2]}") || { warn "No destination mount for ${p[2]}"; rc=1; continue; } label="${p[1]}: volume ${p[2]}"; ((freeze)) && label+=" (consistent pass)" managed_copy "$label" "${p[0]}" "${p[5]}" "$mp" "$freeze" "${p[2]}" || rc=1 done return $rc } transfer_managed_binds() { local v p rc=0 ((CONSOLIDATE && OPT_BINDS)) || return 0 for v in "${MBINDS[@]}"; do IFS=$'\x1f' read -r -a p <<<"$v" managed_copy "${p[1]}: bind ${p[2]}" "${p[0]}" "${p[2]}" "${p[2]}" 0 "" || rc=1 done return $rc } transfer_managed_images() { local s sf sid sname sarch v p refs b64 rc=0 ((CONSOLIDATE && OPT_IMAGES)) || return 0 for s in "${MSERVERS[@]}"; do IFS=$'\x1f' read -r -a sf <<<"$s"; sid=${sf[0]}; sname=${sf[1]}; sarch=${sf[6]}; refs="" if [[ $sarch != "$D_ARCH" ]]; then warn "$sname uses $sarch; its apps will rebuild from their last successful commits for $D_ARCH" continue fi for v in "${MIMGS[@]}"; do IFS=$'\x1f' read -r -a p <<<"$v"; [[ ${p[0]} == "$sid" ]] && refs+="${p[2]} "; done [[ -n $refs ]] || continue b64=$(printf '%s' "$refs" | base64 | tr -d '\r\n') ui_run "Shipping images from $sname" managed_images_stream "$sid" "$b64" || rc=1 done return $rc } managed_images_stream() { rscript SRC "$RS_MANAGED_IMAGES" SID="$1" IMAGES_B64="$2" | rssh DST "$DST_SUDO gzip -dc | $DST_SUDO docker load" local -a ps=("${PIPESTATUS[@]}") [[ ${ps[0]} == 0 && ${ps[1]} == 0 ]] } managed_resource_uuids() { # server id local sid=$1 res ep out="" for res in "${MRES[@]}"; do IFS=$'\x1f' read -r -a ep <<<"$res" [[ ${ep[5]} == "$sid" ]] && out+="${out:+ }${ep[1]}" done printf '%s' "$out" } stop_managed_sources() { local s p uuids b64 rc=0 ((CONSOLIDATE)) || return 0 for s in "${MSERVERS[@]}"; do IFS=$'\x1f' read -r -a p <<<"$s"; uuids=$(managed_resource_uuids "${p[0]}") [[ -n $uuids ]] || { warn "No migrated resources found on ${p[1]}"; continue; } b64=$(printf '%s' "$uuids" | base64 | tr -d '\r\n') ui_run "Stopping migrated workloads on ${p[1]}" rscript SRC "$RS_MANAGED_STOP" SID="${p[0]}" NAMES_B64="$b64" || rc=1 done return $rc } rollback_managed_sources() { local s p rc=0 ((CONSOLIDATE)) || return 0 for s in "${MSERVERS[@]}"; do IFS=$'\x1f' read -r -a p <<<"$s" ui_run "Restarting workloads on ${p[1]}" rscript SRC "$RS_MANAGED_ROLLBACK" SID="${p[0]}" || rc=1 done return $rc } bridge_managed_sources() { local s p body_b64 domains="" domains_b64 rc=0 h ((CONSOLIDATE && OPT_BRIDGE)) || return 0 for h in "${FQDNS[@]}"; do [[ $h == *sslip.io || $h == \** ]] || domains+="$h "; done body_b64=$(printf '%s\n%s\n' "$RS_PRELUDE" "$RS_BRIDGE" | base64 | tr -d '\r\n') domains_b64=$(printf '%s' "$domains" | base64 | tr -d '\r\n') for s in "${MSERVERS[@]}"; do IFS=$'\x1f' read -r -a p <<<"$s" ui_run "Forwarding ${p[1]}:80/443 ${G_ARR} $D_IP until DNS switches" rscript SRC "$RS_MANAGED_BRIDGE" \ SID="${p[0]}" TO="$D_IP" DOMAINS_B64="$domains_b64" BODY_B64="$body_b64" || rc=1 done return $rc } transfer_images() { local refs="" v for v in "${IMGS[@]}"; do refs+="${v%%|*} "; done [[ -z $refs ]] && return 0 if [[ $MODE == direct ]]; then ui_run "Shipping ${#IMGS[@]} images (docker save ${G_ARR} load)" rscript SRC "$RS_SAVE_IMAGES_DIRECT" \ IMGS="$refs" DSSH="$DSSH" DT="$DT" DS="$DST_SUDO" COMP="$COMP" DECOMP="$DECOMP" else ui_run "Shipping ${#IMGS[@]} images via relay" relay_images "$refs" fi } relay_images() { set -f; rssh SRC "$SRC_SUDO docker save $1 | $COMP" | rssh DST "$DST_SUDO sh -c '$DECOMP | docker load'"; } php() { # role phpvar [K=V...] -> runs PHP_HEAD + body inside the coolify container local r=$1 body=$2; shift 2 rscript "$r" "$RS_PHP" PHP="$PHP_HEAD"$'\n'"$body" "$@" } normalize_localhost() { php DST "$PHP_FRESH_SERVER" >>"$LOG" 2>&1 || die "Could not normalize Coolify's localhost server settings" } authorize_localhost_key() { [[ -z $S_LOCAL_PUB ]] && { warn "Couldn't read Coolify's localhost key — validate the server in the new dashboard if deploys fail."; return 0; } # comment stripped: a "coolify" comment would be deleted by the next Coolify install/upgrade rscript DST "$RS_AUTHORIZE" PUB="$(awk '{print $1" "$2}' <<<"$S_LOCAL_PUB")" TUSER="$S_LOCAL_USER" OPTS="" JOURNAL_IT=1 >>"$LOG" 2>&1 && ok "Coolify's localhost SSH key trusted on the new server" } start_resources() { # arg for PHP_START local out line nok=0 nfail=0 reuse=0 [[ -n $S_ARCH && $S_ARCH == "$D_ARCH" ]] && ((OPT_IMAGES)) && reuse=1 ui_capture out "Starting databases, services and apps on the new server" php DST "$PHP_START" ARGS="$1" REUSE_IMAGES="$reuse" printf '%s\n' "$out" >>"$LOG" while IFS= read -r line; do line=$(clean "$line") case $line in OK*) nok=$((nok + 1)); printf ' %s%s%s %s\n' "$GRN" "$G_OK" "$R" "${line#OK }" ;; FAIL*) nfail=$((nfail + 1)); printf ' %s%s%s %s\n' "$RED" "$G_ERR" "$R" "${line#FAIL }" ;; esac done <<<"$out" ((nfail)) && warn "$nfail item(s) failed to start — deploy them from the new dashboard." if ((reuse)); then info "Apps are starting from copied successful images; Coolify builds only if an expected image is missing." else info "Apps are building for the destination CPU from their last successful commits; the old server remains live." fi } rewrite_ip() { ((OPT_IPREWRITE)) && [[ -n $S_IP && -n $D_IP && $S_IP != "$D_IP" ]] || return 0 local out n ui_capture out "Rewriting migrated IP addresses" php DST "$PHP_REWRITE_IP" OLDIP="$S_IP" NEWIP="$D_IP" n=$(printf '%s\n' "$out" | sed -n 's/^REWROTE \([0-9]*\)$/\1/p') [[ -n $n ]] && ok "Rewrote $S_IP ${G_ARR} $D_IP in $n records ${D}(env vars, labels, compose, domains)${R}" || warn "IP rewrite inside Coolify failed (see log) — search your env vars for $S_IP" } remote_servers() { local out line blocked="" name ip port user uuid st sip m ui_capture out "Checking managed-server reachability from the destination" rscript DST "$RS_REMOTE_CHECK" [[ $out == *NOSSH=1* ]] && { warn "No ssh client on the new server — skipping the managed-server check"; return; } while IFS= read -r line; do [[ $line == SRV=* ]] || continue IFS='|' read -r name ip port user uuid st sip <<<"${line#SRV=}" [[ $st == FAIL ]] && { is_ip4 "${sip:-$D_IP}" && blocked+="$name|$ip|$port|$user|$uuid|${sip:-$D_IP}"$'\n'; } done <<<"$out" if [[ -n $blocked ]] && ((OPT_REMOTES)); then info "Some managed servers refuse the new server — allowing it through their firewall via the old server" rscript SRC "$RS_REMOTE_OPEN" LIST="$blocked" 2>>"$LOG" | while IFS= read -r line; do [[ $line == OPEN=* ]] || continue; m=${line##*|}; name=${line#OPEN=}; name=$(clean "${name%%|*}") case $m in ufw|firewalld|iptables) printf ' %s%s%s %s: %s now allows the new server\n' "$GRN" "$G_OK" "$R" "$name" "$m" ;; *) printf ' %s%s%s %s: no host firewall found to change\n' "$D" "$G_NDASH" "$R" "$name" ;; esac done ui_capture out "Rechecking managed-server reachability" rscript DST "$RS_REMOTE_CHECK" fi while IFS= read -r line; do [[ $line == SRV=* ]] || continue IFS='|' read -r name ip port user uuid st sip <<<"${line#SRV=}"; name=$(clean "$name") if [[ $st == OK ]]; then ok "Managed server ${B}$name${R} ($ip) reachable from the new Coolify" else warn "Managed server ${B}$name${R} ($ip:$port) unreachable — allow ${sip:-$D_IP} on port $port in your cloud firewall"; fi done <<<"$out" } # ----------------------------------------------------------------- engines --- EXCL_LIVE="ssh/mux/ source/.env-* source/installation-*.log source/upgrade-*.log" engine_clone() { section "Copy the control plane" quip done_step data1 || { transfer "Coolify files" "$(pairs_data)" "$EXCL_LIVE" || die "Copying /data/coolify failed"; mark data1; } section "Install Coolify ${S_VERSION} on the new server" quip if ! done_step install; then ui_run "Running the official installer (keeps your APP_KEY & secrets)" rscript DST "$RS_INSTALL" URL="$INSTALL_URL" CV="$S_VERSION" INSTALL_SHA256="$INSTALL_SHA256" ui_run "Waiting for Coolify to be healthy" rscript DST "$RS_WAIT_HEALTHY" || die "Coolify didn't become healthy on the new server" mark install fi section "Copy data (old server stays live)" quip copy_data_live section "Restore Coolify's database on the new server" quip ui_run "Dumping Coolify's database (live, consistent snapshot)" rscript SRC "$RS_DUMP" || die "pg_dump failed" transfer "Database dump" "$(rec "database dump" "$RWD/dump" "$RWD/dump" "" "")" || die "Copying the database dump failed" local fixip=0 [[ -n $S_LOCAL_IP && $S_LOCAL_IP != host.docker.internal && $S_LOCAL_IP != 127.0.0.1 && $S_LOCAL_IP != localhost ]] && fixip=1 ui_run "Restoring and restarting Coolify" rscript DST "$RS_RESTORE" URL="$INSTALL_URL" CV="$S_VERSION" INSTALL_SHA256="$INSTALL_SHA256" FIXIP="$fixip" \ OLDIP="$S_IP" NEWIP="$D_IP" PAUSE="$OPT_PAUSE_TASKS" || die "Restore failed" normalize_localhost if ((CONSOLIDATE)); then ui_run "Mapping managed-server workloads onto the new localhost" php DST "$PHP_CONSOLIDATE" || die "Consolidation mapping failed" fi authorize_localhost_key ui_run "Waiting for Coolify to be healthy" rscript DST "$RS_WAIT_HEALTHY" || die "Coolify didn't come back after the restore" rewrite_ip if ((N_REMOTE > 0 && !CONSOLIDATE)); then section "Managed remote servers" warn "Both Coolify instances now manage the same $N_REMOTE remote server(s). Scheduled jobs are paused on the new one until cutover." remote_servers fi if ((OPT_START)); then section "Start on the new server" start_resources "RUNNING:$(running_spec)" fi } engine_import() { section "Prepare Coolify on the new server" quip if [[ $TARGET_KIND == empty ]] && ! done_step install; then ui_run "Running the official installer (Coolify ${S_VERSION})" rscript DST "$RS_INSTALL" URL="$INSTALL_URL" CV="$S_VERSION" INSTALL_SHA256="$INSTALL_SHA256" ui_run "Waiting for Coolify to be healthy" rscript DST "$RS_WAIT_HEALTHY" || die "Coolify didn't become healthy on the new server" php DST "$PHP_FRESH_SERVER" >>"$LOG" 2>&1 mark install else ok "Using the existing Coolify ${D_COOLIFY} on the new server" fi section "Export from the old Coolify" local out line n=0 ui_capture out "Reading selected resources and secrets" php SRC "$PHP_EXPORT" ARGS="$SEL_UUIDS" OUTFILE=export.json || die "Export failed (see log)" while IFS= read -r line; do line=$(clean "$line") case $line in EXPORTED*) n=$((n + 1)); printf ' %s%s%s %s\n' "$GRN" "$G_OK" "$R" "${line#EXPORTED }" ;; WARN*) warn "${line#WARN }" ;; esac done <<<"$out" ((n)) || die "Nothing was exported." transfer "Export file" "$(rec "export" "$RWD/export.json" "$RWD/export.json" "" "")" || die "Copying the export failed" section "Import into the new Coolify" local args="x"; ((OPT_PAUSE_TASKS)) && args+=",pause"; [[ $TARGET_KIND == empty ]] && args+=",rootuser" local ip_old="" ip_new=""; ((OPT_IPREWRITE)) && { ip_old=$S_IP; ip_new=$D_IP; } ui_capture out "Creating resources in the selected destination projects" php DST "$PHP_IMPORT" ARGS="$args" INFILE=export.json OUTFILE=import-rows.json \ OLDIP="$ip_old" NEWIP="$ip_new" TARGET_PROJECT_UUID="$TARGET_PROJECT_UUID" PROJECT_MAP_B64="$(project_map_payload)"; local irc=$? rscript DST "$RS_JOURNAL_ADD" LINES="ROWS import-rows.json" >>"$LOG" 2>&1 local lines="" while IFS= read -r line; do line=$(clean "$line") case $line in IMPORTED*) set -- ${line#IMPORTED }; lines+="UUID $2"$'\n'; printf ' %s%s%s %s\n' "$GRN" "$G_OK" "$R" "${line#IMPORTED }" ;; SKIP*) warn "${line#SKIP }" ;; FAIL*) err "${line#FAIL }" ;; WARN*) warn "${line#WARN }" ;; ROOTUSER*) ok "Admin account copied: ${line#ROOTUSER } ${D}(same password as the old Coolify)${R}" ;; esac done <<<"$out" local u d; for u in $(sel_uuid_list); do for d in applications databases services; do lines+="DIR /data/coolify/$d/$u"$'\n'; done; done rscript DST "$RS_JOURNAL_ADD" LINES="$lines" >>"$LOG" 2>&1 ((irc == 0)) || die "Import failed (see above)" section "Copy data (old server stays live)" quip transfer "Resource files" "$(pairs_resource_dirs)" "" || warn "Some resource files failed to copy" copy_data_live if ((OPT_START)); then section "Start on the new server" local r="" v; for v in "${RUNNING[@]}"; do r+="$v,"; done if [[ -n $r ]]; then start_resources "RUNNING:${r%,}"; else start_resources "$(selected_uuid_csv)"; fi fi } # Two passes: a warm copy while everything runs, then a short frozen pass per volume for consistency. copy_data_live() { ((OPT_VOLUMES && (${#VOLS[@]} || (CONSOLIDATE && ${#MVOLS[@]})))) && { create_volumes || die "Creating volumes failed"; } if ((OPT_VOLUMES && ${#VOLS[@]})) && ! done_step vol1; then transfer "Volumes (warm copy)" "$(pairs_volumes)" && mark vol1; fi if ((OPT_VOLUMES && CONSOLIDATE && ${#MVOLS[@]})) && ! done_step mvol1; then transfer_managed_volumes 0 && mark mvol1 || die "Warm copy from a managed server failed" fi ((OPT_BINDS && ${#BINDS[@]})) && { transfer "Bind folders" "$(pairs_binds)" || warn "Some bind folders failed to copy"; } if ((OPT_BINDS && CONSOLIDATE && ${#MBINDS[@]})); then transfer_managed_binds || warn "Some managed-server bind folders failed to copy"; fi if ((OPT_IMAGES)) && ! done_step images; then transfer_images && mark images || warn "Image transfer failed — apps will be rebuilt instead"; fi if ((OPT_IMAGES && CONSOLIDATE)) && ! done_step mimages; then transfer_managed_images && mark mimages || warn "Some managed-server images failed to transfer — affected apps will be rebuilt" fi if ((OPT_VOLUMES && ${#VOLS[@]})); then transfer "Volumes (consistent pass: each is frozen for seconds)" "$(pairs_volumes freeze)" || die "Copying volume data failed" fi if ((OPT_VOLUMES && CONSOLIDATE && ${#MVOLS[@]})); then transfer_managed_volumes 1 || die "Consistent volume copy from a managed server failed" fi } # ---------------------------------------------------------------- rollback --- auto_rollback() { EXECUTING=0 echo; warn "${B}Something failed — rolling back automatically.${R}" rollback_dest if [[ $PHASE == cutover ]]; then ui_run "Restarting everything on the old server" rscript SRC 'bash "$RWD/rollback.sh"' || err "Old server rollback failed: run $(src_cmd "bash $RWD/rollback.sh")" ((CONSOLIDATE)) && rollback_managed_sources || true else ok "Old server: untouched (nothing was stopped there)" fi } rollback_dest() { [[ $PHASE == cutover ]] && return 0 # a failed cutover keeps the copy; only the old server is restarted local j; j=$(rscript DST 'cat "$RWD/journal" 2>/dev/null' 2>>"$LOG") [[ -z $j ]] && { ok "New server: nothing to undo"; return 0; } if grep -qx 'COOLIFY_PREEXISTED 1' <<<"$j"; then if grep -q '^ROWS ' <<<"$j"; then ui_run "Cancelling imported deployments and build processes" php DST "$PHP_CANCEL_IMPORT_DEPLOYMENTS" INFILE=import-rows.json || { err "Undo stopped before deleting resources because deployment cancellation failed. Retry --undo."; return 1; } ui_run "Removing imported rows from the new Coolify" php DST "$PHP_UNDO_IMPORT" INFILE=import-rows.json fi ui_run "Removing imported containers, volumes and files" rscript DST "$RS_UNDO_IMPORT_FILES" && rscript DST 'rm -f "$RWD/journal" "$RWD/import-rows.json"' >>"$LOG" 2>&1 ok "New server: back to how it was (your existing Coolify is untouched)" else ui_run "Wiping the new server back to empty" rscript DST "$RS_WIPE_DEST" && ok "New server: empty again" fi rm -f "$STATE_FILE" "${PLAN_FILE:-$(state_path).plan}" } # --------------------------------------------------------------- cutover --- run_cutover() { PHASE=cutover EXECUTING=1 local names="" v line section "Stop on the old server" if ((CONSOLIDATE)); then stop_managed_sources || die "Stopping workloads on a managed source server failed" fi if [[ $ENGINE == clone ]]; then names=$(rscript SRC 'docker ps -q --filter label=coolify.managed=true | xargs -r docker inspect -f "{{.Name}}" | sed "s#^/##"' 2>>"$LOG") names="coolify coolify-realtime $(printf '%s ' $names)" else for v in $(sel_uuid_list); do names+="$(rscript SRC "docker ps --format '{{.Names}}' | grep -F -- $(sq "$v")" 2>>"$LOG") "; done fi ui_run "Stopping $(wc -w <<<"$names" | tr -d ' ') container(s) (recorded in rollback.sh)" rscript SRC "$RS_STOP_SOURCE" NAMES="$names" || die "Stopping the old resources failed" section "Final sync" local dstnames=""; for v in "${VOLS[@]}"; do dstnames+="${v%%|*} "; done rscript DST 'for v in $V; do ids=$(docker ps -q --filter volume="$v"); [ -n "$ids" ] && docker stop -t 30 $ids >/dev/null; done; true' V="$dstnames" >>"$LOG" 2>&1 if [[ $ENGINE == clone ]]; then transfer "Coolify files" "$(pairs_data)" "$EXCL_LIVE source/" || die "Final sync of /data/coolify failed" else transfer "Resource files" "$(pairs_resource_dirs)" "" fi ((OPT_VOLUMES && ${#VOLS[@]})) && { transfer "Volumes (final, old resources stopped)" "$(pairs_volumes)" || die "Final volume sync failed"; } ((OPT_BINDS && ${#BINDS[@]})) && transfer "Bind folders (final)" "$(pairs_binds)" if ((CONSOLIDATE)); then ((OPT_VOLUMES && ${#MVOLS[@]})) && transfer_managed_volumes 0 || { ((OPT_VOLUMES && ${#MVOLS[@]})) && die "Final managed-server volume sync failed"; } ((OPT_BINDS && ${#MBINDS[@]})) && transfer_managed_binds || { ((OPT_BINDS && ${#MBINDS[@]})) && die "Final managed-server bind sync failed"; } fi section "Switch on the new server" if [[ $ENGINE == clone ]]; then ui_run "Dumping Coolify's database" rscript SRC "$RS_DUMP" || die "pg_dump failed" transfer "Database dump" "$(rec "database dump" "$RWD/dump" "$RWD/dump" "" "")" || die "Copying the database dump failed" ui_run "Restoring the latest Coolify state" rscript DST "$RS_RESTORE" URL="$INSTALL_URL" CV="$S_VERSION" INSTALL_SHA256="$INSTALL_SHA256" FIXIP=0 OLDIP="$S_IP" NEWIP="$D_IP" PAUSE=0 || die "Restore failed" normalize_localhost if ((CONSOLIDATE)); then ui_run "Mapping managed-server workloads onto the new localhost" php DST "$PHP_CONSOLIDATE" || die "Consolidation mapping failed" fi authorize_localhost_key ui_run "Waiting for Coolify to be healthy" rscript DST "$RS_WAIT_HEALTHY" || die "Coolify didn't come back" rewrite_ip start_resources "RUNNING:$(running_spec)" else ui_run "Re-enabling scheduled tasks and backups" php DST '$d=json_decode(@file_get_contents("/tmp/cm-in.json")?:"{}",true); $n=0; foreach(($d["paused"]??[]) as [$t,$id]) if(preg_match("/^[a-z_]+$/",$t)) $n+=DB::table($t)->where("id",$id)->update(["enabled"=>true]); say("re-enabled $n");' INFILE=import-rows.json start_resources "$(selected_uuid_csv)" fi rscript DST "$RS_RESUME_TASKS" >>"$LOG" 2>&1 if [[ $ENGINE == clone ]] && ((OPT_BRIDGE)) && [[ -n $D_IP && $S_IP != "$D_IP" ]]; then section "Traffic bridge" local doms="" h; for h in "${FQDNS[@]}"; do [[ $h == *sslip.io || $h == \** ]] || doms+="$h "; done ui_run "Forwarding $S_IP:80/443 ${G_ARR} $D_IP until DNS switches" rscript SRC "$RS_BRIDGE" TO="$D_IP" DOMAINS="$doms" GRACE=86400 MAX=604800 || warn "Bridge not started — point DNS to $D_IP as soon as possible" if ((CONSOLIDATE)); then bridge_managed_sources || warn "A managed source server could not start its traffic bridge — update its DNS records immediately" fi fi if ((CONSOLIDATE)); then ui_run "Removing retired source servers from the new dashboard" php DST "$PHP_FINALIZE_CONSOLIDATION" || warn "Workloads moved successfully, but some retired server records remain in the new dashboard and can be removed after verification" fi EXECUTING=0 # Migration complete: setup access is no longer needed. rscript DST "$RS_UNAUTHORIZE" MARKER="$SETUP_MARKER" >>"$LOG" 2>&1 rscript DST 'rm -f "$RWD/journal" "$RWD/import-rows.json"' >>"$LOG" 2>&1 rm -f "$HOME/.ssh/coolify_migrate_pasted" "$STATE_FILE" "${PLAN_FILE:-$(state_path).plan}" report_cutover } # ---------------------------------------------------------------- reports --- report_copy() { local ip=${D_IP:-$DST_HOST} echo; hr printf ' %s%s%s Copy complete%s %s(%s)%s\n' "$B" "$GRN" "$G_PARTY" "$R" "$D" "$(fmt_dur $((SECONDS - RUN_START)))" "$R" hr printf ' %sYour old server is still serving all traffic.%s Nothing there was stopped.\n\n' "$B" "$R" printf ' %sNext%s\n' "$B" "$R" printf ' 1. Test on the new server: dashboard %shttp://%s:8000%s %s(same login as before)%s\n' "$CYN" "$ip" "$R" "$D" "$R" printf ' To try a site before DNS changes, point it at %s in your computer'"'"'s hosts file.\n' "$ip" printf ' 2. When you'"'"'re happy, switch over (final sync of the latest data, then stops the old copies):\n' printf ' %s%s --cutover%s\n' "$GRN" "$(self_cmd)" "$R" printf ' 3. Changed your mind? Remove the copy from the new server:\n %s%s --undo%s\n' "$GRN" "$(self_cmd)" "$R" printf '\n %sLog: %s%s\n' "$D" "$LOG" "$R" credit } report_cutover() { local ip=${D_IP:-$DST_HOST} h echo; hr printf ' %s%s%s Cutover complete%s\n' "$B" "$GRN" "$G_PARTY" "$R" hr if ((${#FQDNS[@]})); then printf ' %sUpdate these DNS records%s %s %s%s%s\n' "$B" "$R" "$G_ARR" "$CYN" "$ip" "$R" for h in "${FQDNS[@]}"; do [[ $h == *sslip.io ]] && continue; printf ' A %-45s %s %s\n' "$h" "$G_ARR" "$ip"; done fi [[ $ENGINE == clone ]] && ((OPT_BRIDGE)) && printf '\n Traffic to the old IP is forwarded to the new server and the forward removes itself once DNS\n points at %s for 24h. Remove it earlier: %s\n' "$ip" "$(src_cmd "docker rm -f cm-bridge")" printf '\n Roll back (restart everything on the old server): %s\n' "$(src_cmd "bash $RWD/rollback.sh")" printf ' When everything works on the new server, you can delete the old one.\n' printf '\n %sLog: %s%s\n' "$D" "$LOG" "$R" credit } credit() { printf '\n %sbuilt by %s%sgrtsnx%s %s %s%shttps://github.com/grtsnx%s\n\n' "$D" "$R" "$B" "$R" "$G_DOT" "$R" "$CYN" "$R"; } # ------------------------------------------------------- background worker --- should_detach() { [[ $DETACH == 1 ]] && return 0 [[ $DETACH == auto ]] && ((SRC_LOCAL)) && return 0 # web terminal may close (cutover stops Coolify itself) return 1 } follow() { # outfile [pid] local out=$1 pid=${2:-} tp rc trap 'printf "\n"; info "Stopped watching — the migration keeps running. Re-attach: --attach"; exit 0' INT tail -n +1 -f "$out" 2>/dev/null & tp=$! while [[ ! -f $out.rc ]]; do [[ -n $pid ]] && ! kill -0 "$pid" 2>/dev/null && { sleep 1; break; } sleep 1 done sleep 1; kill "$tp" 2>/dev/null; wait "$tp" 2>/dev/null rc=$(cat "$out.rc" 2>/dev/null || echo 1) exit "$rc" } attach() { local out pid [[ -f $STATE_DIR/logs/latest.run ]] || die "No background migration found in $STATE_DIR/logs" read -r pid out <"$STATE_DIR/logs/latest.run" [[ $pid =~ ^[0-9]+$ && -f $out ]] || die "Output of the last background migration is gone" if [[ -f $out.rc ]] || ! kill -0 "$pid" 2>/dev/null; then cat "$out"; exit "$(cat "$out.rc" 2>/dev/null || echo 1)"; fi info "Attached to migration (pid $pid). Ctrl-C to stop watching." follow "$out" "$pid" } launch_worker() { # function to run local fn=$1 out="$STATE_DIR/logs/$RUN_ID.out" pid ( umask 077; : >"$out" ); rm -f "$out.rc" ( trap '' HUP INT trap cleanup EXIT trap 'exit 143' TERM INTERACTIVE=0 HANDED_OFF=0 LIVE=1 WORKER_RC_FILE="$out.rc" exec >"$out" 2>&1 "$fn" ) & pid=$! HANDED_OFF=1 ( umask 077; echo "$pid $out" >"$STATE_DIR/logs/latest.run" ) echo ok "Running in a background worker ${D}(pid $pid)${R} — it survives this terminal closing." ((SRC_LOCAL)) && warn "Coolify's web terminal will disconnect when Coolify is stopped. That's expected." info "Watch again later over SSH: ${B}$( [[ -f $SCRIPT_PATH ]] && printf '%s --attach' "$(self_cmd)" || printf 'tail -f %s' "$out")${R}" info "Ctrl-C only stops watching. Abort (and roll back) with: kill $pid" echo follow "$out" "$pid" } check_update() { [[ -n ${COOLIFY_MIGRATE_NO_UPDATE_CHECK:-} ]] && return 0 local latest latest=$(curl -fsSL --max-time 3 "$TOOL_URL" 2>/dev/null | sed -n 's/^TOOL_VERSION="\([0-9.]*\)"$/\1/p' | head -n 1) [[ -z $latest || $latest == "$TOOL_VERSION" ]] && return 0 [[ $(printf '%s\n%s\n' "$TOOL_VERSION" "$latest" | sort -t. -k1,1n -k2,2n -k3,3n | tail -n 1) == "$latest" ]] || return 0 warn "A newer version is available: ${B}v$latest${R} (you have v$TOOL_VERSION). Get it with:" printf ' %scurl -fsSL %s | bash%s\n\n' "$GRN" "$TOOL_URL" "$R" } # ------------------------------------------------------------------- main --- run_copy() { EXECUTING=1 PHASE=copy rscript DST "$RS_JOURNAL_INIT" ENGINE="$ENGINE" OVERWRITE="$OVERWRITE" >>"$LOG" 2>&1 || die "Couldn't write the journal on the new server" if [[ $ENGINE == clone ]]; then engine_clone; else engine_import; fi EXECUTING=0 save_plan report_copy } main() { local arg for arg in "$@"; do [[ $arg == -h || $arg == --help ]] && { usage; exit 0; }; done mkdir -p "$STATE_DIR/logs" "$STATE_DIR/state"; chmod 700 "$STATE_DIR"; ( umask 077; : >"$LOG" ) parse_args "$@" ((ATTACH)) && attach [[ -n $SRC_KEY ]] && SRC_KEY=${SRC_KEY/#\~/$HOME} [[ -n $DST_KEY ]] && DST_KEY=${DST_KEY/#\~/$HOME} [[ $SRC_AUTH == password && -z $SRC_PASS ]] && SRC_PASS=${SRC_PASSWORD:-} [[ $DST_AUTH == password && -z $DST_PASS ]] && DST_PASS=${DST_PASSWORD:-} banner log "coolify-migrate v$TOOL_VERSION on $(uname -sm) bash $BASH_VERSION" check_update command -v ssh >/dev/null || die "ssh client not found" if ((CLEANUP_ORPHANS)); then STEP_T=2 section "Connect" connect_role DST "New server" section "Clean stale deployment queues" ((INTERACTIVE)) && { ui_confirm "Cancel and remove only deployment queues whose application no longer exists?" y || die "Aborted"; } ui_run "Cancelling orphan deployment helpers and processes" php DST "$PHP_CLEANUP_ORPHAN_DEPLOYMENTS" || die "Orphan deployment cleanup failed" ok "Only deployment queues without an application were removed; valid applications and deployments were untouched." credit; exit 0 fi if ((!SRC_LOCAL)) && [[ -z $SRC_HOST && -d /data/coolify && $(uname -s) == Linux ]] && ((INTERACTIVE)); then ui_confirm "Coolify found on this server. Migrate ${B}from this server${R}?" y && SRC_LOCAL=1 fi if ((SRC_LOCAL)); then SRC_USER=$(id -un) SRC_HOST=$(hostname) SRC_PORT=22 SRC_AUTH=local; fi # --cutover / --undo pick up the saved plan for this destination if ((CUTOVER || UNDO)); then if [[ -z $DST_HOST ]]; then load_only_plan else load_plan || die "No copy in progress for $DST_HOST found." fi validate_loaded_plan ((SRC_LOCAL)) && { SRC_USER=$(id -un) SRC_HOST=$(hostname); } STATE_FILE=${PLAN_FILE%.plan} PHASE=$([[ $CUTOVER == 1 ]] && echo cutover || echo undo) STEP_T=$((UNDO ? 2 : 7)) section "Connect" connect_role SRC "Old server"; connect_role DST "New server" if ((UNDO)); then section "Undo the copy on the new server" ((INTERACTIVE)) && { ui_confirm "Remove the copy from $(target DST)? (the old server is not touched)" n || die "Aborted"; } PHASE=copy; rollback_dest; credit; exit 0 fi section "Transfer link" setup_transfer section "Ready to switch" info "This will: stop the migrated resources on the old server, copy the very latest data, start everything on the new server." warn "Downtime = the final data sync + start. Everything is recorded so it can be rolled back automatically." ((INTERACTIVE)) && { ui_confirm "Cut over now?" n || die "Aborted"; } should_detach && { launch_worker run_cutover; exit 0; } run_cutover; exit 0 fi STEP_T=9 section "Connect" connect_role SRC "Old server" connect_role DST "New server" section "Discover" info "Inspecting both servers${G_ELL}" discover ok "Coolify ${B}${S_VERSION:-?}${R} $G_DOT $S_COUNTS" section "Plan" choose_what choose_target_project narrow_to_selection ok "${#VOLS[@]} volumes $G_DOT ${#BINDS[@]} bind folders $G_DOT ${#IMGS[@]} images $G_DOT ${#FQDNS[@]} domains" choose_options check_destination_capacity "$(estimated_bytes)" || die "Destination capacity check failed; resize the new server before migrating." show_plan ((PLAN_ONLY)) && { info "Plan only — nothing changed."; exit 0; } ((INTERACTIVE)) && { ui_confirm "Start copying?" y || die "Aborted by user"; } STATE_FILE="$(state_path)" if [[ -s $STATE_FILE ]]; then warn "Found an unfinished copy to this server." ui_confirm "Resume it (skip finished steps)?" y || : >"$STATE_FILE" fi : >>"$STATE_FILE" section "Transfer link" setup_transfer should_detach && { launch_worker run_copy; exit 0; } run_copy } [[ ${BASH_SOURCE[0]} == "$0" ]] && main "$@"