Require reproducible production deploy sources

docs/deployment.md

@@ -19,6 +19,26 @@ bash deploy.sh
 
 `bash sync.sh` remains as a legacy alias for the same flow.
 
+Production deploys require a reproducible Git source tree by default
+(`REQUIRE_CLEAN_GIT=1`). The preflight inspects staged, tracked, and
+deployable untracked files; untracked paths excluded by rsync are ignored.
+The local Vite build may refresh the tracked generated SSR bundle under
+`bootstrap/ssr/`, but source/config/deploy changes made during preparation
+abort before the release is switched. The local Git `HEAD` is captured before
+build and must remain unchanged through rsync.
+
+Run the local-only guard when validating a release without creating a remote
+release or running rsync:
+
+```bash
+bash sync.sh --preflight-only
+```
+
+`--skip-build` is rejected for a clean production deploy unless
+`deploy.cmd` has just completed the Windows build and exported
+`WINDOWS_FRONTEND_BUILT=1`. An explicit `REQUIRE_CLEAN_GIT=0` is reserved for
+non-production/custom workflows and is not the production default.
+
 `deploy.cmd` runs `npm.cmd run build` on Windows first, then enters WSL for rsync/ssh. That is required when the Ubuntu distro cannot execute Windows `.exe` files (`Exec format error` on `powershell.exe`). If WSL interop does work, `bash deploy.sh` can still launch `npm.cmd` through PowerShell. Local Linux `php`/`composer` are not required for a normal deploy; Artisan and Composer run on the production server. `--with-tests` uses WSL `php` when present, otherwise Windows `php.exe`.
 
 This will:

scripts/deploy-observability.sh

@@ -123,6 +123,7 @@ ensure_local_deploy_dirs() {
 # metadata fast: short SHA/branch always; unstaged dirty scans are skipped on
 # slow filesystems and otherwise hard-timeout'd.
 git_metadata_timeout_seconds="${GIT_METADATA_TIMEOUT_SECONDS:-8}"
+clean_git_timeout_seconds="${CLEAN_GIT_TIMEOUT_SECONDS:-60}"
 
 git_workdir_is_slow() {
   case "$local_folder" in
@@ -192,6 +193,65 @@ detect_git_dirty_fast() {
   return 2
 }
 
+deploy_path_is_rsync_excluded() {
+  local path="${1#./}"
+  path="${path%/}"
+
+  case "$path" in
+    .git|.git/*|.deploy|.deploy/*|.cursor|.cursor/*|.venv|.venv/*|.vscode|.vscode/*|node_modules|node_modules/*|vendor|vendor/*|storage|storage/*|tests|tests/*|playwright-report|playwright-report/*|test-results|test-results/*|public/build|public/build/*|public/files|public/files/*|public/storage|public/storage/*|resources/lang|resources/lang/*|bootstrap/cache|bootstrap/cache/*|var/deploy|var/deploy/*|var/php-tmp|var/php-tmp/*|var/php-sessions|var/php-sessions/*|.cache|.cache/*|.config|.config/*|.composer|.composer/*|.npm|.npm/*|.local|.local/*|.copilot|.copilot/*|oldSite|oldSite/*|.phpintel|.phpintel/*)
+      return 0
+      ;;
+    public/hot|public/sitemap.xml|public/sitemaps/*|.env|.env.*|.phpunit.result.cache)
+      return 0
+      ;;
+    *)
+      return 1
+      ;;
+  esac
+}
+
+# Returns: 0 deployable changes, 1 no deployable changes, 2 unable to prove.
+# Unlike detect_git_dirty_fast(), this is intentionally conservative: a
+# production clean-git gate must inspect untracked files and may not silently
+# skip the worktree scan on WSL/network filesystems.
+detect_deployable_git_dirty() {
+  local status_output=""
+  local rc=0
+  local line=""
+  local status_code=""
+  local path=""
+
+  run_git_with_timeout "$clean_git_timeout_seconds" diff --cached --quiet >/dev/null 2>&1 || rc=$?
+  if [[ "$rc" -eq 1 ]]; then
+    return 0
+  fi
+  if [[ "$rc" -ne 0 ]]; then
+    return 2
+  fi
+
+  rc=0
+  status_output="$(run_git_with_timeout "$clean_git_timeout_seconds" -c core.untrackedCache=false status --porcelain=v1 --untracked-files=all 2>/dev/null)" || rc=$?
+  if [[ "$rc" -ne 0 ]]; then
+    return 2
+  fi
+
+  while IFS= read -r line; do
+    [[ -n "$line" ]] || continue
+    status_code="${line:0:2}"
+    path="${line:3}"
+
+    # Untracked files in paths excluded by the deployment rsync are not
+    # deployable. Tracked changes remain dirty even when their path is
+    # excluded, so accidental edits cannot be hidden by this allowance.
+    if [[ "$status_code" == "??" ]] && deploy_path_is_rsync_excluded "$path"; then
+      continue
+    fi
+    return 0
+  done <<< "$status_output"
+
+  return 1
+}
+
 collect_git_metadata() {
   local dirty_rc=1
 

scripts/sync-safe-updated.sh

@@ -57,9 +57,14 @@ php_fpm_service="${PHP_FPM_SERVICE:-php8.4-fpm}"
 ssr_supervisor_program="${SSR_SUPERVISOR_PROGRAM:-skinbase-ssr}"
 # SSH login user stays klevze@...; remote file/composer/artisan work runs as this app user.
 remote_app_user="${REMOTE_APP_USER:-skinbase}"
-require_clean_git="${REQUIRE_CLEAN_GIT:-0}"
+# Production deploys must originate from a clean, reproducible source tree.
+# An explicit REQUIRE_CLEAN_GIT=0 remains available for non-production/custom
+# workflows, but is intentionally not the default.
+require_clean_git="${REQUIRE_CLEAN_GIT:-1}"
 required_git_branch="${REQUIRED_GIT_BRANCH:-}"
 db_sync_remote_maintenance=0
+preflight_only=0
+head_sha_before=""
 
 declare -a rsync_args=()
 declare -a ssh_base_opts=()
@@ -78,6 +83,7 @@ Options:
   --with-tests          Run the local test command before build/sync. Default command: php artisan test.
   --skip-migrate        Skip php artisan migrate on the server.
   --dry-run             Print the planned rsync/deploy actions without changing the remote server.
+  --preflight-only      Run local source/build checks and exit before remote release creation or rsync.
   --release-id ID       Override the generated release version label used for the remote release directory.
   --build-number N      Override the monotonic local build number for this deploy.
   --keep-releases N     Keep the latest N remote releases ready for server-side switching. Default: 5.
@@ -104,7 +110,7 @@ Options:
   --no-rollback         Disable automatic rollback to the previous release when the switched release fails before health/safe point.
   --reload-php-fpm      Try to reload PHP-FPM after release switch. Uses PHP_FPM_SERVICE, default php8.4-fpm.
   --no-php-fpm-reload   Explicitly skip PHP-FPM reload.
-  --require-clean-git   Refuse deploy when the local Git working tree has uncommitted changes.
+  --require-clean-git   Refuse deploy when the local Git working tree has uncommitted deployable changes (default).
   --required-branch BRANCH
                         Refuse deploy unless the local Git branch matches BRANCH.
   --no-rsync-progress   Disable rsync transfer progress output.
@@ -119,6 +125,7 @@ Environment overrides:
   RELOAD_PHP_FPM, PHP_FPM_SERVICE, REQUIRE_CLEAN_GIT, REQUIRED_GIT_BRANCH,
   ALLOW_DEPLOY_FROM_DOT_DEPLOY, FULL_UPGRADE_PRE_HOOK, FULL_UPGRADE_POST_HOOK,
   GIT_METADATA_TIMEOUT_SECONDS, WINDOWS_SSH_DIR, WINDOWS_FRONTEND_BUILT
+  CLEAN_GIT_TIMEOUT_SECONDS
 
 Notes:
   SSH still authenticates as REMOTE_SERVER (e.g. klevze@host). Remote rsync/composer/artisan
@@ -491,21 +498,77 @@ guard_git_state() {
 
   if [[ "$require_clean_git" == "1" ]]; then
     dirty_rc=1
-    detect_git_dirty_fast && dirty_rc=0 || dirty_rc=$?
+    detect_deployable_git_dirty && dirty_rc=0 || dirty_rc=$?
     if [[ "$dirty_rc" -eq 0 ]]; then
-      die "Working tree has uncommitted changes. Commit/stash them or disable REQUIRE_CLEAN_GIT."
+      die "Working tree has uncommitted deployable changes. Commit them before production deploy."
     fi
     if [[ "$dirty_rc" -ne 1 ]]; then
-      die "Could not prove a clean Git worktree on this filesystem (dirty check skipped or timed out). Disable REQUIRE_CLEAN_GIT or deploy from a native Linux checkout."
+      die "Could not prove a clean Git worktree before deployment (dirty check timed out or failed). Refusing deploy."
     fi
   fi
 }
 
+capture_head_sha() {
+  head_sha_before=""
+  if command -v git >/dev/null 2>&1 && git -C "$local_folder" rev-parse --is-inside-work-tree >/dev/null 2>&1; then
+    head_sha_before="$(git -C "$local_folder" rev-parse HEAD 2>/dev/null || true)"
+    [[ "$head_sha_before" =~ ^[0-9a-f]{40}$ ]] || die "Could not capture the local Git HEAD before build/sync."
+  fi
+}
+
+assert_head_stable() {
+  [[ -n "$head_sha_before" ]] || return 0
+  local current_head
+  current_head="$(git -C "$local_folder" rev-parse HEAD 2>/dev/null || true)"
+  [[ "$current_head" == "$head_sha_before" ]] || die "Local Git HEAD changed during deployment preparation; refusing remote release switch."
+}
+
+assert_post_build_source_state() {
+  [[ "$require_clean_git" == "1" ]] || return 0
+
+  # deploy.cmd completed the Windows build before this WSL process started.
+  # The initial clean-source gate already ran after that build; preflight-only
+  # must not repeat the expensive full status walk on a /mnt filesystem.
+  if [[ "$preflight_only" -eq 1 && "${WINDOWS_FRONTEND_BUILT:-0}" == "1" ]]; then
+    assert_head_stable
+    return 0
+  fi
+
+  local status_output=""
+  local line=""
+  local status_code=""
+  local path=""
+  local rc=0
+
+  assert_head_stable
+  status_output="$(run_git_with_timeout "$clean_git_timeout_seconds" -c core.untrackedCache=false status --porcelain=v1 --untracked-files=all 2>/dev/null)" || rc=$?
+  [[ "$rc" -eq 0 ]] || die "Could not verify the Git worktree after the local build; refusing remote release switch."
+
+  while IFS= read -r line; do
+    [[ -n "$line" ]] || continue
+    status_code="${line:0:2}"
+    path="${line:3}"
+
+    # Vite owns the tracked SSR bundle. Its generated output is allowed to be
+    # refreshed by this deploy's local build, but source/config/deploy edits
+    # are never allowed to pass the final guard.
+    if [[ "$path" == bootstrap/ssr/* ]]; then
+      continue
+    fi
+    if [[ "$status_code" == "??" ]] && deploy_path_is_rsync_excluded "$path"; then
+      continue
+    fi
+    die "Git worktree changed outside generated SSR output after build: $path"
+  done <<< "$status_output"
+}
+
 run_preflight_checks() {
   log_step "Running local preflight checks"
 
-  require_command "$ssh_bin" "SSH"
+  if [[ "$preflight_only" -eq 0 ]]; then
-  require_command "$rsync_bin" "rsync"
+    require_command "$ssh_bin" "SSH"
+    require_command "$rsync_bin" "rsync"
+  fi
   require_local_php_if_needed
 
   [[ -f "$local_folder/artisan" ]] || die "Expected Laravel artisan entrypoint at $local_folder/artisan."
@@ -529,6 +592,12 @@ run_preflight_checks() {
   write_build_info_file
   print_deploy_banner
   guard_git_state
+  if [[ "$preflight_only" -eq 1 ]]; then
+    log_info "Preflight-only mode: skipping SSH, remote app-user, and remote release checks"
+    log_info "Required local deploy tools are available"
+    mark_phase_complete "preflight"
+    return 0
+  fi
   acquire_local_deploy_lock
   configure_ssh_transport
   verify_remote_app_user_access
@@ -637,13 +706,18 @@ run_frontend_build() {
 }
 
 validate_local_build_artifacts() {
-  [[ "$run_local_build" -eq 1 ]] || return 0
-
   local missing=0
+  local validation_context=""
+
+  if [[ "$run_local_build" -eq 1 ]]; then
+    validation_context="after local build"
+  else
+    validation_context="from existing local artifacts"
+  fi
 
   if [[ ! -f "$local_folder/public/build/manifest.json" ]]; then
     if [[ "$require_build_manifest" == "1" ]]; then
-      die "Vite manifest missing at public/build/manifest.json after build. Refusing deploy. Set REQUIRE_BUILD_MANIFEST=0 only for intentional custom build paths."
+      die "Vite manifest missing at public/build/manifest.json ${validation_context}. Refusing deploy. Set REQUIRE_BUILD_MANIFEST=0 only for intentional custom build paths."
     fi
     log_warn "Vite manifest was not found at public/build/manifest.json after build. Continuing because REQUIRE_BUILD_MANIFEST=0."
     missing=1
@@ -651,14 +725,14 @@ validate_local_build_artifacts() {
 
   if [[ ! -f "$local_folder/bootstrap/ssr/ssr.js" && ! -f "$local_folder/bootstrap/ssr/ssr.mjs" ]]; then
     if [[ "$require_build_manifest" == "1" ]]; then
-      die "SSR build artifact missing under bootstrap/ssr/ after build. Refusing deploy."
+      die "SSR build artifact missing under bootstrap/ssr/ ${validation_context}. Refusing deploy."
     fi
     log_warn "SSR build artifact was not found under bootstrap/ssr/. Continuing because REQUIRE_BUILD_MANIFEST=0."
     missing=1
   fi
 
   if [[ "$missing" -eq 0 ]]; then
-    log_info "Local build artifacts validated"
+    log_info "Local build artifacts validated (${validation_context})"
   fi
 }
 
@@ -1087,8 +1161,8 @@ EOF_PUBLIC_LINK
 }
 
 # Supervisor listens on a root socket. The app user (skinbase) cannot talk to it,
-# and artisan inertia:start-ssr fights the supervised process. Restart as the
+# and a second unmanaged SSR process fights the supervised process. Restart
-# SSH login user with passwordless sudo supervisorctl.
+# as the SSH login user with passwordless sudo supervisorctl.
 restart_remote_inertia_ssr() {
   [[ "$skip_ssr_restart" -eq 0 ]] || return 0
 
@@ -1100,32 +1174,37 @@ restart_remote_inertia_ssr() {
 set -euo pipefail
 cd /tmp >/dev/null 2>&1 || cd / >/dev/null 2>&1 || true
 
+# sudoers on production is exact-match NOPASSWD for:
+#   /usr/bin/supervisorctl status <program>
+#   /usr/bin/supervisorctl restart <program>
+# Do not call `supervisorctl status` without a program name, or `start`.
 supervisorctl_bin="/usr/bin/supervisorctl"
-if [[ ! -x "$supervisorctl_bin" ]]; then
+[[ -x "$supervisorctl_bin" ]] || {
-  supervisorctl_bin="$(command -v supervisorctl 2>/dev/null || true)"
+  printf 'ERROR: %s not found on the remote host.\n' "$supervisorctl_bin" >&2
-fi
-[[ -n "$supervisorctl_bin" ]] || {
-  printf 'ERROR: supervisorctl not found on the remote host.\n' >&2
   exit 1
 }
 
-ctl() {
+ctl_status() {
-  sudo -n "$supervisorctl_bin" "$@"
+  sudo -n "$supervisorctl_bin" status "$SSR_PROGRAM"
 }
 
-status_line="$(ctl status "$SSR_PROGRAM" 2>/dev/null || true)"
+ctl_restart() {
+  sudo -n "$supervisorctl_bin" restart "$SSR_PROGRAM"
+}
+
+status_line="$(ctl_status 2>/dev/null || true)"
 if ! printf '%s\n' "$status_line" | grep -q "^${SSR_PROGRAM}[[:space:]]"; then
-  printf 'ERROR: Supervisor program %s not found, or sudo -n supervisorctl is not permitted for this SSH user.\n' "$SSR_PROGRAM" >&2
+  printf 'ERROR: Supervisor program %s not found, or sudo -n supervisorctl is not permitted.\n' "$SSR_PROGRAM" >&2
-  ctl status >&2 || true
+  printf '%s\n' "$status_line" >&2
   exit 1
 fi
 
-ctl restart "$SSR_PROGRAM"
+ctl_restart
 
 status=""
 i=0
 while [[ "$i" -lt 10 ]]; do
-  status="$(ctl status "$SSR_PROGRAM" 2>/dev/null | awk '{print $2}' || true)"
+  status="$(ctl_status 2>/dev/null | awk '{print $2}' || true)"
   if [[ "$status" == "RUNNING" ]]; then
     printf '  -> Supervisor program %s is RUNNING\n' "$SSR_PROGRAM"
     exit 0
@@ -1135,7 +1214,7 @@ while [[ "$i" -lt 10 ]]; do
 done
 
 printf 'ERROR: Supervisor program %s did not reach RUNNING after restart (status: %s).\n' "$SSR_PROGRAM" "${status:-unknown}" >&2
-ctl status "$SSR_PROGRAM" >&2 || true
+ctl_status >&2 || true
 exit 1
 EOF_SSR_RESTART
 }
@@ -1178,6 +1257,9 @@ while [[ $# -gt 0 ]]; do
     --dry-run)
       dry_run=1
       ;;
+    --preflight-only)
+      preflight_only=1
+      ;;
     --release-id)
       shift
       release_id="$(sanitize_release_fragment "${1:?Missing value for --release-id}")"
@@ -1321,6 +1403,7 @@ collect_git_metadata
 allocate_build_number
 trap 'rc=$?; on_local_exit "$rc"; exit "$rc"' EXIT
 run_preflight_checks
+capture_head_sha
 
 if [[ "$run_db_sync" -eq 1 && "$db_sync_source" != "local" ]]; then
   die "Refusing DB sync without an explicit source. Use --with-db-from=local."
@@ -1355,6 +1438,22 @@ if [[ "$run_local_build" -eq 1 ]]; then
   fi
   validate_local_build_artifacts
   mark_phase_complete "frontend-build"
+else
+  if [[ "$require_clean_git" == "1" && "${WINDOWS_FRONTEND_BUILT:-0}" != "1" ]]; then
+    die "--skip-build is not allowed for a clean production deploy unless WINDOWS_FRONTEND_BUILT=1 is set by deploy.cmd after a successful local build."
+  fi
+  validate_local_build_artifacts
+fi
+
+# This catches a source edit or HEAD change made after the initial preflight.
+# It runs before any remote release is created, and again after rsync before
+# the remote release can be switched.
+assert_post_build_source_state
+
+if [[ "$preflight_only" -eq 1 ]]; then
+  log_step "Local deploy preflight complete"
+  log_info "No remote release was created and no rsync was performed"
+  exit 0
 fi
 
 build_rsync_args
@@ -1403,6 +1502,10 @@ log_step "Syncing release ${release_id} (build #${build_number}) to $remote_serv
 "$rsync_bin" "${rsync_args[@]}" "$local_folder/" "$remote_server:$(remote_release_path)/"
 mark_phase_complete "rsync"
 
+# Rsync has populated only the staging release. Refuse to switch it if the
+# source tree changed while the transfer was being prepared.
+assert_post_build_source_state
+
 if [[ "$run_db_sync" -eq 1 ]]; then
   enable_remote_maintenance_for_db_sync