#!/usr/bin/env bash # Shared deploy observability helpers sourced by scripts/sync-safe-updated.sh. # Keeps build numbers, local history, and progress logging out of the main script body. : "${root_dir:?root_dir must be set before sourcing deploy-observability.sh}" : "${local_folder:?local_folder must be set before sourcing deploy-observability.sh}" build_number="${BUILD_NUMBER:-${build_number:-}}" local_deploy_dir="${LOCAL_DEPLOY_DIR:-$root_dir/var/deploy}" local_deploy_log_dir="${local_deploy_dir}/logs" local_build_number_file="${local_deploy_dir}/build-number" local_history_file="${local_deploy_dir}/history.jsonl" local_latest_file="${local_deploy_dir}/latest.json" local_deploy_log_file="${local_deploy_log_file:-}" build_info_file="${build_info_file:-}" git_sha="${git_sha:-unknown}" git_branch="${git_branch:-unknown}" git_dirty="${git_dirty:-0}" deploy_started_epoch="${deploy_started_epoch:-$(date +%s)}" deploy_started_utc="${deploy_started_utc:-$(date -u +%Y-%m-%dT%H:%M:%SZ)}" deploy_started_local="${deploy_started_local:-$(date +%Y-%m-%d\ %H:%M:%S\ %Z)}" deploy_finished_epoch="${deploy_finished_epoch:-}" deploy_status="${deploy_status:-running}" deploy_phase_current="${deploy_phase_current:-0}" phase_started_epoch="${phase_started_epoch:-$deploy_started_epoch}" previous_release_meta="${previous_release_meta:-}" rsync_show_progress="${rsync_show_progress:-1}" declare -a deploy_phase_timings=("${deploy_phase_timings[@]+"${deploy_phase_timings[@]}"}") format_duration() { local total_seconds="${1:-0}" local hours minutes seconds if ! [[ "$total_seconds" =~ ^[0-9]+$ ]]; then total_seconds=0 fi hours=$((total_seconds / 3600)) minutes=$(((total_seconds % 3600) / 60)) seconds=$((total_seconds % 60)) if (( hours > 0 )); then printf '%dh %02dm %02ds' "$hours" "$minutes" "$seconds" elif (( minutes > 0 )); then printf '%dm %02ds' "$minutes" "$seconds" else printf '%ds' "$seconds" fi } elapsed_since_start() { local now now="$(date +%s)" format_duration "$((now - deploy_started_epoch))" } json_escape() { local value="${1-}" value="${value//\\/\\\\}" value="${value//\"/\\\"}" value="${value//$'\n'/\\n}" value="${value//$'\r'/\\r}" value="${value//$'\t'/\\t}" printf '%s' "$value" } append_deploy_log() { local line="$1" [[ -n "${local_deploy_log_file:-}" ]] || return 0 printf '%s\n' "$line" >> "$local_deploy_log_file" 2>/dev/null || true } obs_log_step() { local message="$1" local now local line now="$(date +%s)" deploy_phase_current=$((deploy_phase_current + 1)) phase_started_epoch="$now" line="$(printf '[step %s] (+%s) %s' "$deploy_phase_current" "$(elapsed_since_start)" "$message")" printf '\n%s\n' "$line" append_deploy_log "$(date -u +%Y-%m-%dT%H:%M:%SZ) STEP ${line}" } obs_log_info() { local message="$1" printf ' -> %s\n' "$message" append_deploy_log "$(date -u +%Y-%m-%dT%H:%M:%SZ) INFO ${message}" } obs_log_warn() { local message="$1" printf 'WARN: %s\n' "$message" >&2 append_deploy_log "$(date -u +%Y-%m-%dT%H:%M:%SZ) WARN ${message}" } obs_die() { printf 'ERROR: %s\n' "$1" >&2 append_deploy_log "$(date -u +%Y-%m-%dT%H:%M:%SZ) ERROR $1" exit 1 } mark_phase_complete() { local label="${1:-phase}" local now local elapsed now="$(date +%s)" elapsed=$((now - phase_started_epoch)) deploy_phase_timings+=("${label}=$(format_duration "$elapsed")") obs_log_info "Phase finished in $(format_duration "$elapsed") (total $(elapsed_since_start))" } ensure_local_deploy_dirs() { mkdir -p "$local_deploy_dir" "$local_deploy_log_dir" } # Full-tree dirty checks (git diff / status / describe --dirty) can take minutes # on WSL /mnt/ checkouts because every file is stat'd over 9p. Keep deploy # metadata fast: short SHA/branch always; unstaged dirty scans are skipped on # slow filesystems and otherwise hard-timeout'd. git_metadata_timeout_seconds="${GIT_METADATA_TIMEOUT_SECONDS:-8}" git_workdir_is_slow() { case "$local_folder" in /mnt/[a-zA-Z]/*|/mnt/[a-zA-Z]) return 0 ;; esac local fstype="" if command -v findmnt >/dev/null 2>&1; then fstype="$(findmnt -n -o FSTYPE --target "$local_folder" 2>/dev/null || true)" case "$fstype" in 9p|drvfs|cifs|nfs|nfs4|fuse|fuseblk|fuse.*) return 0 ;; esac fi return 1 } run_git_with_timeout() { local timeout_seconds="${1:-$git_metadata_timeout_seconds}" shift # Do not use timeout --foreground: that leaves git children unkilled. # -k 2 sends SIGKILL if SIGTERM cannot interrupt a stuck 9p stat. if command -v timeout >/dev/null 2>&1; then timeout -k 2 "$timeout_seconds" git -C "$local_folder" "$@" return $? fi git -C "$local_folder" "$@" } # Returns: 0 dirty, 1 clean, 2 timeout/unknown, 3 skipped unstaged scan (slow FS). detect_git_dirty_fast() { local status_output="" local rc=0 # Staged changes are cheap (index vs HEAD, no worktree walk). run_git_with_timeout 3 diff --cached --quiet >/dev/null 2>&1 || rc=$? if [[ "$rc" -eq 1 ]]; then return 0 fi if [[ "$rc" -eq 124 || "$rc" -eq 137 ]]; then return 2 fi if [[ "$rc" -ne 0 ]]; then return 2 fi if git_workdir_is_slow; then return 3 fi # Porcelain still stats the worktree; only run it on native filesystems. rc=0 status_output="$(run_git_with_timeout "$git_metadata_timeout_seconds" -c core.untrackedCache=false status --porcelain=v1 -uno 2>/dev/null)" || rc=$? if [[ "$rc" -eq 0 ]]; then if [[ -n "${status_output//[[:space:]]/}" ]]; then return 0 fi return 1 fi return 2 } collect_git_metadata() { local dirty_rc=1 git_sha="unknown" git_branch="unknown" git_dirty=0 if ! command -v git >/dev/null 2>&1 || ! git -C "$local_folder" rev-parse --is-inside-work-tree >/dev/null 2>&1; then return 0 fi printf ' -> Collecting local Git metadata (SHA/branch; dirty check is bounded)...\n' git_sha="$(git -C "$local_folder" rev-parse --short HEAD 2>/dev/null || printf 'unknown')" git_branch="$(git -C "$local_folder" branch --show-current 2>/dev/null || printf 'DETACHED')" # Capture without tripping set -e (clean/timeout/skip are non-zero). dirty_rc=1 detect_git_dirty_fast && dirty_rc=0 || dirty_rc=$? if [[ "$dirty_rc" -eq 0 ]]; then git_dirty=1 elif [[ "$dirty_rc" -eq 3 ]]; then git_dirty=0 printf ' -> Skipping unstaged Git dirty scan on this filesystem (WSL /mnt or network mount); using staged-only check.\n' >&2 elif [[ "$dirty_rc" -eq 2 ]]; then git_dirty=0 printf ' -> Git dirty check timed out after %ss on this filesystem; continuing without dirty marker.\n' \ "$git_metadata_timeout_seconds" >&2 else git_dirty=0 fi } peek_next_build_number() { local existing="" local next=1 if [[ -f "$local_build_number_file" ]]; then existing="$(tr -d '[:space:]' < "$local_build_number_file" 2>/dev/null || true)" if [[ "$existing" =~ ^[0-9]+$ ]]; then next=$((existing + 1)) fi fi printf '%s' "$next" } allocate_build_number() { local existing="" local persist=1 ensure_local_deploy_dirs # Dry-runs never consume the counter unless an explicit override is provided for labeling only. if [[ "${dry_run:-0}" -eq 1 ]]; then persist=0 fi if [[ -n "$build_number" ]]; then [[ "$build_number" =~ ^[1-9][0-9]*$ ]] || obs_die "Build number must be a positive integer. Received: $build_number" else build_number="$(peek_next_build_number)" fi [[ "$build_number" =~ ^[1-9][0-9]*$ ]] || obs_die "Build number must be a positive integer. Received: $build_number" if [[ "$persist" -eq 1 ]]; then if [[ -f "$local_build_number_file" ]]; then existing="$(tr -d '[:space:]' < "$local_build_number_file" 2>/dev/null || true)" if [[ "$existing" =~ ^[0-9]+$ ]] && (( build_number < existing )); then : else printf '%s\n' "$build_number" > "$local_build_number_file" fi else printf '%s\n' "$build_number" > "$local_build_number_file" fi fi } setup_local_deploy_log_file() { local stamp ensure_local_deploy_dirs stamp="$(date -u +%Y%m%d-%H%M%S)" local_deploy_log_file="${local_deploy_log_dir}/deploy-${stamp}-b${build_number}.log" { printf 'Skinbase production deploy log\n' printf 'started_at_utc=%s\n' "$deploy_started_utc" printf 'started_at_local=%s\n' "$deploy_started_local" printf 'build_number=%s\n' "$build_number" printf 'release_id=%s\n' "${release_id:-pending}" printf 'mode=%s\n' "${deploy_mode:-normal}" printf 'source=%s\n' "$local_folder" printf 'target=%s:%s\n' "${remote_server:-}" "${remote_folder:-}" printf 'git_sha=%s\n' "$git_sha" printf 'git_branch=%s\n' "$git_branch" printf 'git_dirty=%s\n' "$git_dirty" printf '%s\n' '---' } > "$local_deploy_log_file" } write_build_info_file() { build_info_file="${local_folder}/build-info.json" cat > "$build_info_file" < 0 )); then timings_json="[" first=1 for timing in "${deploy_phase_timings[@]}"; do if (( first )); then first=0 else timings_json+="," fi timings_json+="\"$(json_escape "$timing")\"" done timings_json+="]" fi cat > "$local_latest_file" <> "$local_history_file" printf '\n' >> "$local_history_file" } print_deploy_banner() { local dirty_marker="" local dry_marker="" if [[ "${git_dirty:-0}" -eq 1 ]]; then dirty_marker=" [dirty]" fi if [[ "${dry_run:-0}" -eq 1 ]]; then dry_marker=" (dry-run preview)" fi printf '\n' printf '============================================================\n' printf ' Skinbase production deploy\n' printf '============================================================\n' printf ' Build number : %s%s\n' "${build_number:-pending}" "$dry_marker" printf ' Release : %s\n' "${release_id:-pending}" printf ' Mode : %s\n' "${deploy_mode:-normal}" printf ' Started : %s\n' "$deploy_started_local" printf ' Source : %s\n' "$local_folder" printf ' Target : %s:%s\n' "${remote_server:-}" "${remote_folder:-}" printf ' Git : %s (%s)%s\n' "$git_sha" "$git_branch" "$dirty_marker" printf ' Local log : %s\n' "${local_deploy_log_file:-n/a}" printf '============================================================\n' } finalize_local_deploy_observability() { local exit_code="${1:-0}" local final_status="success" if [[ "$exit_code" -ne 0 ]]; then final_status="failed" elif [[ "${dry_run:-0}" -eq 1 ]]; then final_status="dry-run" fi if [[ -n "${build_number:-}" && "$deploy_status" == "running" ]]; then write_local_deploy_history "$final_status" "${previous_release_meta:-}" if [[ -n "${local_deploy_log_file:-}" ]]; then { printf '%s\n' '---' printf 'status=%s\n' "$final_status" printf 'exit_code=%s\n' "$exit_code" printf 'finished_at_utc=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" printf 'duration=%s\n' "$(elapsed_since_start)" printf 'release_id=%s\n' "${release_id:-}" printf 'build_number=%s\n' "${build_number:-}" } >> "$local_deploy_log_file" 2>/dev/null || true fi fi } release_id_with_build() { local timestamp local vcs_fragment="manual" if [[ -n "${release_id:-}" ]]; then printf '%s' "$release_id" return 0 fi timestamp="$(date -u +%Y%m%d-%H%M%S)" # Never use `git describe --dirty` here: --dirty walks the whole work tree and # can hang for minutes on WSL /mnt/* checkouts. Prefer already-collected SHA. if [[ -n "${git_sha:-}" && "$git_sha" != "unknown" ]]; then vcs_fragment="$git_sha" if [[ "${git_dirty:-0}" -eq 1 ]]; then vcs_fragment="${vcs_fragment}-dirty" fi elif command -v git >/dev/null 2>&1 && git -C "$local_folder" rev-parse --is-inside-work-tree >/dev/null 2>&1; then vcs_fragment="$(git -C "$local_folder" rev-parse --short HEAD 2>/dev/null || printf 'manual')" fi vcs_fragment="$(sanitize_release_fragment "$vcs_fragment")" [[ -n "$vcs_fragment" ]] || vcs_fragment="manual" if [[ -n "${build_number:-}" ]]; then printf '%s-b%s-%s' "$timestamp" "$build_number" "$vcs_fragment" else printf '%s-%s' "$timestamp" "$vcs_fragment" fi }