QSfera/Server/scripts/verify-rpi-production.sh

#!/usr/bin/env sh
set -eu

COMPOSE_FILE="${COMPOSE_FILE:-docker-compose.rpi5.yml}"
ENV_FILE="${ENV_FILE:-.env}"
SERVICE="${SERVICE:-qsfera-cloud-server}"
CONTAINER="${CONTAINER:-qsfera-cloud-server}"
EXPECTED_IMAGE="${EXPECTED_IMAGE:-qsfera-cloud-server:7.0.0-rpi5-stable}"
EXPECTED_VERSION="${EXPECTED_VERSION:-7.0.0}"
EXPECTED_EDITION="${EXPECTED_EDITION:-stable}"
LOCAL_STATUS_URL="${LOCAL_STATUS_URL:-https://127.0.0.1:9200/status.php}"
PUBLIC_STATUS_URL="${PUBLIC_STATUS_URL:-}"
METRICS_URL="${METRICS_URL:-http://127.0.0.1:9205/metrics}"
REQUIRE_METRICS="${REQUIRE_METRICS:-false}"
VERIFY_IMAGE_VERSION="${VERIFY_IMAGE_VERSION:-true}"
ROOT_PATH="${ROOT_PATH:-/}"
DATA_PATH="${DATA_PATH:-/mnt/data}"
MIN_ROOT_AVAILABLE_KB="${MIN_ROOT_AVAILABLE_KB:-5242880}"
MIN_DATA_AVAILABLE_KB="${MIN_DATA_AVAILABLE_KB:-20971520}"
EXPECTED_DOCKER_ROOT_PREFIX="${EXPECTED_DOCKER_ROOT_PREFIX:-/mnt/data}"
STRICT_DOCKER_ROOT="${STRICT_DOCKER_ROOT:-false}"

errors=0
warnings=0
tmp_dir="$(mktemp -d)"

cleanup() {
    rm -rf "$tmp_dir"
}
trap cleanup EXIT INT TERM

ok() {
    echo "OK: $*"
}

warn() {
    warnings=$((warnings + 1))
    echo "WARN: $*" >&2
}

error() {
    errors=$((errors + 1))
    echo "ERROR: $*" >&2
}

is_true() {
    case "$(printf '%s' "$1" | tr '[:upper:]' '[:lower:]')" in
        true|1|yes|on)
            return 0
            ;;
        *)
            return 1
            ;;
    esac
}

require_command() {
    if command -v "$1" >/dev/null 2>&1; then
        ok "Command available: $1"
    else
        error "Required command is missing: $1"
    fi
}

compose() {
    if [ -n "$ENV_FILE" ] && [ -f "$ENV_FILE" ]; then
        docker compose --env-file "$ENV_FILE" -f "$COMPOSE_FILE" "$@"
    else
        docker compose -f "$COMPOSE_FILE" "$@"
    fi
}

available_kb() {
    df -Pk "$1" 2>/dev/null | awk 'NR == 2 { print $4 }'
}

format_gib() {
    awk -v kb="$1" 'BEGIN { printf "%.1f", kb / 1048576 }'
}

check_disk() {
    path="$1"
    minimum_kb="$2"
    label="$3"
    current_kb="$(available_kb "$path")"

    if [ -z "$current_kb" ]; then
        error "Cannot read free disk space for $label at $path."
        return
    fi

    if [ "$current_kb" -ge "$minimum_kb" ]; then
        ok "$label free space $(format_gib "$current_kb") GiB >= $(format_gib "$minimum_kb") GiB"
    else
        error "$label free space $(format_gib "$current_kb") GiB < $(format_gib "$minimum_kb") GiB"
    fi
}

check_status_url() {
    label="$1"
    url="$2"
    curl_opts="$3"
    output="$tmp_dir/status.json"

    if curl $curl_opts "$url" -o "$output"; then
        ok "$label returned a successful HTTP response"
    else
        error "$label request failed: $url"
        return
    fi

    if python3 - "$output" "$EXPECTED_EDITION" "$EXPECTED_VERSION" "$label" <<'PY'
import json
import sys

path, expected_edition, expected_version, label = sys.argv[1:]
with open(path, "r", encoding="utf-8") as handle:
    payload = json.load(handle)

checks = [
    ("installed", payload.get("installed") is True, True, payload.get("installed")),
    ("maintenance", payload.get("maintenance") is False, False, payload.get("maintenance")),
    ("needsDbUpgrade", payload.get("needsDbUpgrade") is False, False, payload.get("needsDbUpgrade")),
    ("edition", payload.get("edition") == expected_edition, expected_edition, payload.get("edition")),
    ("productversion", payload.get("productversion") == expected_version, expected_version, payload.get("productversion")),
]

failed = False
for name, passed, expected, actual in checks:
    if passed:
        print(f"OK: {label} {name} = {actual}")
    else:
        print(f"ERROR: {label} {name}: expected {expected!r}, got {actual!r}", file=sys.stderr)
        failed = True

if failed:
    sys.exit(1)
PY
    then
        ok "$label JSON invariants match expected production state"
    else
        error "$label JSON invariants failed"
    fi
}

check_metrics() {
    output="$tmp_dir/metrics.txt"

    if [ -n "${METRICS_BEARER_TOKEN:-}" ]; then
        metrics_result=0
        curl -fsS -H "Authorization: Bearer ${METRICS_BEARER_TOKEN}" "$METRICS_URL" -o "$output" || metrics_result=$?
    else
        metrics_result=0
        curl -fsS "$METRICS_URL" -o "$output" || metrics_result=$?
    fi

    if [ "$metrics_result" -eq 0 ]; then
        if grep -Eq '^(# HELP|# TYPE|go_|process_)' "$output"; then
            ok "Metrics endpoint returned Prometheus-formatted content"
        else
            error "Metrics endpoint responded, but content does not look like Prometheus metrics."
        fi
        return
    fi

    if is_true "$REQUIRE_METRICS"; then
        error "Metrics endpoint is required but unavailable: $METRICS_URL"
    else
        warn "Metrics endpoint unavailable: $METRICS_URL"
    fi
}

for command_name in docker curl python3 awk df; do
    require_command "$command_name"
done

if [ ! -f "$COMPOSE_FILE" ]; then
    error "Compose file not found: $COMPOSE_FILE"
fi

if [ -n "$ENV_FILE" ] && [ ! -f "$ENV_FILE" ]; then
    error "Environment file not found: $ENV_FILE"
fi

if [ "$errors" -eq 0 ]; then
    if compose config >/dev/null; then
        ok "Docker Compose config renders successfully for $COMPOSE_FILE"
    else
        error "Docker Compose config failed for $COMPOSE_FILE"
    fi
fi

if docker image inspect "$EXPECTED_IMAGE" >/dev/null 2>&1; then
    ok "Expected image exists locally: $EXPECTED_IMAGE"
else
    error "Expected image is missing locally: $EXPECTED_IMAGE"
fi

if is_true "$VERIFY_IMAGE_VERSION"; then
    version_output="$tmp_dir/image-version.txt"
    if docker run --rm --entrypoint /usr/bin/qsfera "$EXPECTED_IMAGE" version --skip-services >"$version_output" 2>&1; then
        if grep -q "Version: $EXPECTED_VERSION" "$version_output" && grep -q "Edition: $EXPECTED_EDITION" "$version_output"; then
            ok "Image version metadata matches $EXPECTED_EDITION $EXPECTED_VERSION"
        else
            error "Image version metadata does not match expected $EXPECTED_EDITION $EXPECTED_VERSION."
            sed 's/^/  /' "$version_output" >&2
        fi
    else
        error "Cannot read image version metadata from $EXPECTED_IMAGE."
        sed 's/^/  /' "$version_output" >&2
    fi
fi

container_status="$(docker inspect -f '{{.State.Status}}' "$CONTAINER" 2>/dev/null || true)"
if [ "$container_status" = "running" ]; then
    ok "Container $CONTAINER is running"
else
    error "Container $CONTAINER status is '${container_status:-missing}', expected running."
fi

container_health="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$CONTAINER" 2>/dev/null || true)"
case "$container_health" in
    healthy)
        ok "Container $CONTAINER health is healthy"
        ;;
    none)
        warn "Container $CONTAINER has no Docker healthcheck result."
        ;;
    *)
        error "Container $CONTAINER health is '${container_health:-missing}', expected healthy."
        ;;
esac

container_image="$(docker inspect -f '{{.Config.Image}}' "$CONTAINER" 2>/dev/null || true)"
if [ "$container_image" = "$EXPECTED_IMAGE" ]; then
    ok "Container image is $EXPECTED_IMAGE"
else
    error "Container image is '${container_image:-missing}', expected $EXPECTED_IMAGE."
fi

docker_root="$(docker info --format '{{.DockerRootDir}}' 2>/dev/null || true)"
case "$docker_root" in
    "$EXPECTED_DOCKER_ROOT_PREFIX"|"$EXPECTED_DOCKER_ROOT_PREFIX"/*)
        ok "Docker root is under $EXPECTED_DOCKER_ROOT_PREFIX: $docker_root"
        ;;
    "")
        error "Cannot read Docker root directory."
        ;;
    *)
        if is_true "$STRICT_DOCKER_ROOT"; then
            error "Docker root is $docker_root, expected path under $EXPECTED_DOCKER_ROOT_PREFIX."
        else
            warn "Docker root is $docker_root, not under $EXPECTED_DOCKER_ROOT_PREFIX; keep image/build retention active or move data-root during maintenance."
        fi
        ;;
esac

check_disk "$ROOT_PATH" "$MIN_ROOT_AVAILABLE_KB" "Root filesystem"
check_disk "$DATA_PATH" "$MIN_DATA_AVAILABLE_KB" "Data filesystem"
check_status_url "Local server status" "$LOCAL_STATUS_URL" "-kfsS"

if [ -n "$PUBLIC_STATUS_URL" ]; then
    check_status_url "Public server status" "$PUBLIC_STATUS_URL" "-fsS"
fi

check_metrics

if [ "$errors" -gt 0 ]; then
    echo "QSfera Raspberry Pi production verification failed: $errors error(s), $warnings warning(s)." >&2
    exit 1
fi

echo "QSfera Raspberry Pi production verification passed: $warnings warning(s)."