import json
import ssl
import time
from pathlib import Path
from typing import Any, Dict, Optional, Tuple
from urllib.error import HTTPError
from urllib.request import Request, urlopen

import pytest
from conftest import (
    Cluster,
    Instance,
    Retriable,
    TarantoolError,
)

USERNAME = "test_auth"
PASSWORD = "test_auth1A!"


def create_user(instance: Instance):
    try:
        instance.sql(f"CREATE USER \"{USERNAME}\" WITH PASSWORD '{PASSWORD}' USING chap-sha1 OPTION (TIMEOUT = 3.0);")
    except Exception as e:
        if not isinstance(e, TarantoolError):
            raise e


def set_jwt_enabled(instance: Instance, enabled: bool):
    try:
        instance.sql("ALTER SYSTEM RESET jwt_secret" if enabled else "ALTER SYSTEM SET jwt_secret = ''")
    except Exception as e:
        if not isinstance(e, TarantoolError):
            raise e


def authorize(instance: Instance, username: str, password) -> Tuple[Dict[str, str], int]:
    create_user(instance)
    http_listen = instance.http_listen
    request = Request(
        f"http://{http_listen}/api/v1/session",
        data=json.dumps({"username": username, "password": password}).encode(),
        method="POST",
    )
    try:
        with urlopen(request) as response:
            response_data = response.read().decode()
            result = json.loads(response_data)
            return result, response.status
    except HTTPError as e:
        response_data = e.read().decode("utf-8")
        result = json.loads(response_data)
        return result, e.status  # type: ignore


def get_auth_token(instance: Instance) -> str:
    return authorize(instance, USERNAME, PASSWORD)[0]["auth"]


@pytest.fixture
def auth_token(request, instance: Instance) -> Optional[str]:
    auth_type = request.param
    if auth_type == "unauthorized":
        set_jwt_enabled(instance, False)
        return None

    set_jwt_enabled(instance, True)
    return get_auth_token(instance)


def get_authorized(url: str, auth_token: str) -> Any:
    request = Request(url, headers={"Authorization": f"Bearer {auth_token}"}, method="GET")
    return urlopen(request)


def get_unauthorized(url: str) -> Any:
    request = Request(url, method="GET")
    return urlopen(request)


def get_url(url: str, auth_token: Optional[str]) -> Any:
    if auth_token is None:
        return get_unauthorized(url)

    return get_authorized(url, auth_token)


@pytest.mark.webui
def test_http_routes(instance: Instance):
    instance.eval(
        """
        pico.httpd:route({path = '/hello', method = 'GET'}, function(req)
            return {
                status = 200,
                body = 'world'
            }
        end)
        """
    )
    http_listen = instance.http_listen
    with urlopen(f"http://{http_listen}/hello") as response:
        assert response.read() == b"world"


@pytest.mark.webui
@pytest.mark.parametrize("auth_token", ["authorized", "unauthorized"], indirect=True)
def test_webui_basic(instance: Instance, auth_token: Optional[str]):
    http_listen = instance.http_listen

    instance_version = instance.eval("return pico.PICODATA_VERSION")
    instance_slab = instance.call("box.slab.info")
    capacity_usage = round(instance_slab["quota_used"] * 100 / instance_slab["quota_size"], 1)

    with urlopen(f"http://{http_listen}/") as response:
        assert response.headers.get("content-type") == "text/html"

    with get_url(f"http://{http_listen}/api/v1/tiers", auth_token) as response:
        assert response.headers.get("content-type") == "application/json"
        assert json.load(response) == [
            {
                "replicasets": [
                    {
                        "state": "Online",
                        "version": instance_version,
                        "instances": [
                            {
                                "failureDomain": {},
                                "isLeader": True,
                                "currentState": "Online",
                                "targetState": "Online",
                                "name": "default_1_1",
                                "version": instance_version,
                                "httpAddress": http_listen,
                                "binaryAddress": instance.iproto_listen,
                                "pgAddress": instance.pg_listen,
                            }
                        ],
                        "instanceCount": 1,
                        "capacityUsage": capacity_usage,
                        "memory": {
                            "usable": instance_slab["quota_size"],
                            "used": instance_slab["quota_used"],
                        },
                        "uuid": instance.replicaset_uuid(),
                        "name": "default_1",
                    }
                ],
                "replicasetCount": 1,
                "rf": 1,
                "bucketCount": 3000,
                "instanceCount": 1,
                "can_vote": True,
                "name": "default",
                "services": [],
                "capacityUsage": capacity_usage,
                "memory": {
                    "usable": instance_slab["quota_size"],
                    "used": instance_slab["quota_used"],
                },
            }
        ]

    with get_url(f"http://{http_listen}/api/v1/cluster", auth_token) as response:
        assert response.headers.get("content-type") == "application/json"
        assert json.load(response) == {
            "capacityUsage": capacity_usage,
            "clusterName": instance.cluster_name,
            "clusterVersion": instance_version,
            "replicasetsCount": 1,
            "instancesCurrentStateOffline": 0,
            "currentInstaceVersion": instance_version,
            "memory": {
                "usable": instance_slab["quota_size"],
                "used": instance_slab["quota_used"],
            },
            "instancesCurrentStateOnline": 1,
            "plugins": [],
        }

    # Test behavior when HTTP address is missing from _pico_peer_address.
    # This documents expected behavior for clusters upgrading from pre-26.1.
    raft_id = instance.sql(f"SELECT raft_id FROM _pico_instance WHERE name = '{instance.name}'")[0][0]
    instance.sql(f"DELETE FROM _pico_peer_address WHERE raft_id = {raft_id} AND connection_type = 'http'")

    with get_url(f"http://{http_listen}/api/v1/tiers", auth_token) as response:
        tiers = json.load(response)
        http_address = tiers[0]["replicasets"][0]["instances"][0]["httpAddress"]
        assert http_address == "", "httpAddress should be empty when not in _pico_peer_address"


@pytest.mark.webui
def test_webui_with_plugin(cluster: Cluster):
    cluster_cfg = """
    cluster:
        name: test
        tier:
            red:
                replication_factor: 1
            blue:
                replication_factor: 1
            green:
                replication_factor: 1
    """
    cluster.set_config_file(yaml=cluster_cfg)

    plugin_1 = "testplug"
    plugin_1_services = ["testservice_1", "testservice_2"]
    plugin_2 = "testplug_small"
    plugin_2_service = "testservice_1"
    version_1 = "0.1.0"

    i1 = cluster.add_instance(wait_online=True, tier="red")
    i2 = cluster.add_instance(wait_online=True, tier="blue")
    i3 = cluster.add_instance(wait_online=True, tier="green")

    i1.call("pico.install_plugin", plugin_1, version_1)
    i1.call("pico.install_plugin", plugin_2, version_1)
    i1.call(
        "pico.service_append_tier",
        plugin_1,
        version_1,
        plugin_1_services[0],
        "red",
    )
    i1.call(
        "pico.service_append_tier",
        plugin_1,
        version_1,
        plugin_1_services[1],
        "blue",
    )
    i1.call(
        "pico.service_append_tier",
        plugin_2,
        version_1,
        plugin_2_service,
        "blue",
    )

    http_listen = i1.http_listen
    instance_version = i1.eval("return pico.PICODATA_VERSION")

    with urlopen(f"http://{http_listen}/") as response:
        assert response.headers.get("content-type") == "text/html"

    create_user(i1)
    auth_token = get_auth_token(i1)

    instance_template = {
        "failureDomain": {},
        "isLeader": True,
        "currentState": "Online",
        "targetState": "Online",
        "version": instance_version,
    }
    instance_1 = {
        **instance_template,
        "name": "red_1_1",
        "binaryAddress": i1.iproto_listen,
        "pgAddress": i1.pg_listen,
        "httpAddress": i1.http_listen,
    }
    instance_2 = {
        **instance_template,
        "name": "blue_1_1",
        "binaryAddress": i2.iproto_listen,
        "pgAddress": i2.pg_listen,
        "httpAddress": i2.http_listen,
    }
    instance_3 = {
        **instance_template,
        "name": "green_1_1",
        "binaryAddress": i3.iproto_listen,
        "pgAddress": i3.pg_listen,
        "httpAddress": i3.http_listen,
    }

    replicaset_template = {
        "state": "Online",
        "version": instance_version,
        "instanceCount": 1,
        "capacityUsage": 0.0,
        "memory": {
            "usable": 67108864,
            "used": 0,
        },
        "uuid": i1.replicaset_uuid(),
        "name": "r1",
    }
    r1 = {
        **replicaset_template,
        "uuid": i1.replicaset_uuid(),
        "name": "red_1",
        "instances": [instance_1],
    }
    r2 = {
        **replicaset_template,
        "uuid": i2.replicaset_uuid(),
        "name": "blue_1",
        "instances": [instance_2],
    }
    r3 = {
        **replicaset_template,
        "uuid": i3.replicaset_uuid(),
        "name": "green_1",
        "instances": [instance_3],
    }

    tier_template = {
        "replicasetCount": 1,
        "rf": 1,
        "bucketCount": 3000,
        "instanceCount": 1,
        "can_vote": True,
        "capacityUsage": 0.0,
        "memory": {
            "usable": 67108864,
            "used": 0,
        },
    }

    tier_red = {
        **tier_template,
        "name": "red",
        "services": [plugin_1_services[0]],
        "replicasets": [r1],
    }
    tier_blue = {
        **tier_template,
        "name": "blue",
        "services": [plugin_1_services[1], plugin_2_service],
        "replicasets": [r2],
    }
    tier_green = {**tier_template, "name": "green", "services": [], "replicasets": [r3]}

    with get_url(f"http://{http_listen}/api/v1/tiers", auth_token) as response:
        assert response.headers.get("content-type") == "application/json"
        assert sorted(json.load(response), key=lambda tier: tier["name"]) == [
            tier_blue,
            tier_green,
            tier_red,
        ]

    with get_url(f"http://{http_listen}/api/v1/cluster", auth_token) as response:
        assert response.headers.get("content-type") == "application/json"
        assert json.load(response) == {
            "capacityUsage": 0.0,
            "clusterName": cluster.id,
            "replicasetsCount": 3,
            "instancesCurrentStateOnline": 3,
            "instancesCurrentStateOffline": 0,
            "currentInstaceVersion": instance_version,
            "clusterVersion": instance_version,
            "memory": {"usable": 201326592, "used": 0},
            "plugins": [
                plugin_1 + " " + version_1,
                plugin_2 + " " + version_1,
            ],
        }


@pytest.mark.webui
def test_webui_replicaset_state(cluster: Cluster):
    cluster_cfg = """
    cluster:
        name: test
        tier:
            red:
                replication_factor: 2
    """
    cluster.set_config_file(yaml=cluster_cfg)

    i1 = cluster.add_instance(wait_online=True, tier="red")
    i2 = cluster.add_instance(wait_online=True, tier="red")
    i3 = cluster.add_instance(wait_online=True, tier="red")
    i4 = cluster.add_instance(wait_online=True, tier="red")

    # 1. Make i3 the leader via transfer
    # 2. Kill i3
    # 3. Wait for a new leader to be elected
    # 4. Check that replicaset status == Online
    leader = cluster.leader()
    leader.raft_transfer_leadership(i3.raft_id)
    i3.kill()
    cluster.wait_has_states(i3, "Offline", "Offline")

    # With check_quorum, advance raft clock to trigger immediate election.
    for i in [i1, i2, i4]:
        i.call("pico.raft_tick", 100)

    cluster.wait_leader_elected([i1, i2, i4])

    # we have to query i4 to avoid incosistency in cluster view
    # data from global tables (as instance state etc) could be
    # distributed in cluster with delay significant enough to
    # cause flaks in tests
    http_listen = i4.http_listen
    instance_version = i1.eval("return pico.PICODATA_VERSION")

    with urlopen(f"http://{http_listen}/") as response:
        assert response.headers.get("content-type") == "text/html"

    create_user(i4)
    auth_token = get_auth_token(i4)

    instance_template = {
        "failureDomain": {},
        "currentState": "Online",
        "targetState": "Online",
        "version": instance_version,
    }
    instance_1 = {
        **instance_template,
        "name": "red_1_1",
        "isLeader": True,
        "binaryAddress": i1.iproto_listen,
        "pgAddress": i1.pg_listen,
        "httpAddress": i1.http_listen,
    }
    instance_2 = {
        **instance_template,
        "name": "red_1_2",
        "isLeader": False,
        "binaryAddress": i2.iproto_listen,
        "pgAddress": i2.pg_listen,
        "httpAddress": i2.http_listen,
    }
    instance_3 = {
        **instance_template,
        "name": "red_2_1",
        "isLeader": False,
        "currentState": "Offline",
        "targetState": "Offline",
        "binaryAddress": i3.iproto_listen,
        "pgAddress": i3.pg_listen,
        # HTTP address is read from storage, so available even when instance is offline
        "httpAddress": i3.http_listen,
    }
    instance_4 = {
        **instance_template,
        "name": "red_2_2",
        "isLeader": True,
        "binaryAddress": i4.iproto_listen,
        "pgAddress": i4.pg_listen,
        "httpAddress": i4.http_listen,
    }

    replicaset_template = {
        "state": "Online",
        "version": instance_version,
        "instanceCount": 2,
        "capacityUsage": 0.0,
    }
    r1 = {
        **replicaset_template,
        "state": "Online",
        "uuid": i1.replicaset_uuid(),
        "name": "red_1",
        "instances": [instance_1, instance_2],
        "memory": {
            "usable": 67108864,
            "used": 0,
        },
    }
    r2 = {
        **replicaset_template,
        "state": "Online",
        # use i4, as i3 is already dead
        "uuid": i4.replicaset_uuid(),
        "name": "red_2",
        "instances": [instance_3, instance_4],
        "memory": {
            "usable": 67108864,
            "used": 0,
        },
    }

    tier_red = {
        "replicasetCount": 2,
        "rf": 2,
        "bucketCount": 3000,
        "instanceCount": 4,
        "can_vote": True,
        "name": "red",
        "services": [],
        "replicasets": [r1, r2],
        "capacityUsage": 0.0,
        "memory": {
            # r1.memory.usable + r2.memory.usable
            "usable": 134217728,
            "used": 0,
        },
    }

    with get_url(f"http://{http_listen}/api/v1/tiers", auth_token) as response:
        assert response.headers.get("content-type") == "application/json"
        assert sorted(json.load(response), key=lambda tier: tier["name"]) == [
            tier_red,
        ], "/api/v1/tier"

    with get_url(f"http://{http_listen}/api/v1/cluster", auth_token) as response:
        assert response.headers.get("content-type") == "application/json"
        assert json.load(response) == {
            "capacityUsage": 0.0,
            "clusterName": cluster.id,
            "replicasetsCount": 2,
            "instancesCurrentStateOnline": 3,
            "instancesCurrentStateOffline": 1,
            "currentInstaceVersion": instance_version,
            "clusterVersion": instance_version,
            "memory": {"usable": 134217728, "used": 0},
            "plugins": [],
        }, "/api/v1/cluster"


@pytest.mark.webui
def test_webui_can_vote_flag(cluster: Cluster):
    cluster_cfg = """
    cluster:
        name: test
        tier:
            red:
                replication_factor: 1
            blue:
                replication_factor: 1
                can_vote: false
    """
    cluster.set_config_file(yaml=cluster_cfg)

    i1 = cluster.add_instance(wait_online=True, tier="red")
    i2 = cluster.add_instance(wait_online=True, tier="blue")

    http_listen = i1.http_listen
    instance_version = i1.eval("return pico.PICODATA_VERSION")

    with urlopen(f"http://{http_listen}/") as response:
        assert response.headers.get("content-type") == "text/html"

    create_user(i1)
    auth_token = get_auth_token(i1)

    instance_template = {
        "failureDomain": {},
        "isLeader": True,
        "currentState": "Online",
        "targetState": "Online",
        "version": instance_version,
    }
    instance_1 = {
        **instance_template,
        "name": "red_1_1",
        "binaryAddress": i1.iproto_listen,
        "pgAddress": i1.pg_listen,
        "httpAddress": i1.http_listen,
    }
    instance_2 = {
        **instance_template,
        "name": "blue_1_1",
        "binaryAddress": i2.iproto_listen,
        "pgAddress": i2.pg_listen,
        "httpAddress": i2.http_listen,
    }

    replicaset_template = {
        "state": "Online",
        "version": instance_version,
        "instanceCount": 1,
        "capacityUsage": 0.0,
        "memory": {
            "usable": 67108864,
            "used": 0,
        },
        "uuid": i1.replicaset_uuid(),
        "name": "r1",
    }
    r1 = {
        **replicaset_template,
        "uuid": i1.replicaset_uuid(),
        "name": "red_1",
        "instances": [instance_1],
    }
    r2 = {
        **replicaset_template,
        "uuid": i2.replicaset_uuid(),
        "name": "blue_1",
        "instances": [instance_2],
    }

    tier_template = {
        "replicasetCount": 1,
        "rf": 1,
        "bucketCount": 3000,
        "instanceCount": 1,
        "capacityUsage": 0.0,
        "memory": {
            "usable": 67108864,
            "used": 0,
        },
    }

    tier_red = {
        **tier_template,
        "name": "red",
        "services": [],
        "replicasets": [r1],
        "can_vote": True,
    }
    tier_blue = {
        **tier_template,
        "can_vote": False,
        "name": "blue",
        "services": [],
        "replicasets": [r2],
    }

    with get_url(f"http://{http_listen}/api/v1/tiers", auth_token) as response:
        assert response.headers.get("content-type") == "application/json"
        assert sorted(json.load(response), key=lambda tier: tier["name"]) == [
            tier_blue,
            tier_red,
        ]


@pytest.mark.webui
def test_webui_https(cluster: Cluster):
    https_certs_path = Path(__file__).parent.parent / "https_certs"
    cluster_cfg = f"""
    cluster:
        name: test
        tier:
            red:
                replication_factor: 1
    instance:
        http:
            tls:
                enabled: true
                key_file: {https_certs_path / "key.pem"}
                cert_file: {https_certs_path / "cert.pem"}
                password_file: {https_certs_path / "password.txt"}
    """
    cluster.set_config_file(yaml=cluster_cfg)

    i1 = cluster.add_instance(wait_online=True, tier="red", enable_http=True)

    https_listen = i1.http_listen

    # Create contxext to validate SSL over self-signed cert
    context = ssl.create_default_context()
    context.load_verify_locations(cafile=https_certs_path / "cert.pem")

    with urlopen(f"https://{https_listen}/", context=context) as response:
        assert response.headers.get("content-type") == "text/html"


@pytest.mark.webui
def test_jwt_session_login_success(instance: Instance):
    response_data, status_code = authorize(instance, USERNAME, PASSWORD)

    assert status_code == 200
    assert "auth" in response_data
    assert "refresh" in response_data
    assert isinstance(response_data["auth"], str)
    assert isinstance(response_data["refresh"], str)
    assert len(response_data["auth"].split(".")) == 3
    assert len(response_data["refresh"].split(".")) == 3


@pytest.mark.webui
def test_jwt_session_login_invalid_credentials(instance: Instance):
    response_data, status_code = authorize(instance, USERNAME, "wrongpass")

    assert status_code == 401
    assert response_data["error"] == "wrongCredentials"
    assert response_data["errorMessage"] == "invalid credentials"

    response_data, status_code = authorize(instance, "someuser", "wrongpass")

    assert status_code == 401
    assert response_data["error"] == "wrongCredentials"
    assert response_data["errorMessage"] == "invalid credentials"


@pytest.mark.webui
def test_jwt_session_refresh_success(instance: Instance):
    http_listen = instance.http_listen

    response_data, _ = authorize(instance, USERNAME, PASSWORD)
    refresh_token = response_data["refresh"]

    with get_authorized(f"http://{http_listen}/api/v1/session", refresh_token) as response:
        new_tokens = json.loads(response.read().decode())
        status_code = response.status

    assert status_code == 200

    assert "auth" in new_tokens
    assert "refresh" in new_tokens
    assert isinstance(new_tokens["auth"], str)
    assert isinstance(new_tokens["refresh"], str)
    assert len(new_tokens["auth"].split(".")) == 3
    assert len(new_tokens["refresh"].split(".")) == 3


@pytest.mark.webui
@pytest.mark.parametrize("auth_token", ["authorized", "unauthorized"], indirect=True)
def test_ui_config(instance: Instance, auth_token):
    http_listen = instance.http_listen

    with get_url(f"http://{http_listen}/api/v1/config", auth_token) as response:
        assert response.headers.get("content-type") == "application/json"
        assert response.status == 200
        assert json.loads(response.read().decode()) == {"isAuthEnabled": auth_token is not None}


def test_healthcheck_k8s_probes(cluster: Cluster):
    """
    Test K8s health check probes (startup, live, ready) lifecycle.
    They are designed to comply with k8s specs:
    https://kubernetes.io/docs/concepts/configuration/liveness-readiness-startup-probes/

    Test flow:
    1. Deploy 2 instances with RF=3 -> startup 503 (replicaset not ready)
    2. Reduce RF to 2 -> startup 200 (RF met)
    3. Verify live and ready return 200
    4. Put i1 Offline -> startup 200 (latched), ready 503
    """
    # Configure cluster with RF=3
    cluster.set_config_file(
        yaml=f"""
cluster:
    name: {cluster.id}
    tier:
        default:
            replication_factor: 3
"""
    )

    # Step 1: Deploy 2 instances - RF=3 so replicaset is not ready
    i1, i2 = cluster.deploy(instance_count=2, tier="default", enable_http=True)
    http_listen = i1.http_listen
    base_url = f"http://{http_listen}/api/v1/health"

    # Startup probe should return 503 (replicaset not ready - only 2 of 3)
    try:
        with get_unauthorized(f"{base_url}/startup") as response:
            assert False, f"Expected 503, got {response.status}"
    except HTTPError as e:
        assert e.code == 503, f"Expected 503, got {e.code}"
        body = json.loads(e.read())
        assert body["status"] == "not_ready"
        assert "replicaset" in body["reason"].lower()

    # Step 2: Reduce RF to 2 to make replicaset ready
    i1.sql("UPDATE _pico_tier SET replication_factor = 2")

    # Wait for startup to return 200
    deadline = time.monotonic() + 10
    startup_ok = False
    while time.monotonic() < deadline:
        try:
            with get_unauthorized(f"{base_url}/startup") as response:
                if response.status == 200:
                    startup_ok = True
                    break
        except HTTPError:
            pass
        time.sleep(0.2)
    assert startup_ok, "Startup probe did not return 200 after RF was met"

    # Step 3: Check that live and ready also return 200
    with get_unauthorized(f"{base_url}/live") as response:
        assert response.status == 200, f"Expected live to return 200, got {response.status}"

    with get_unauthorized(f"{base_url}/ready") as response:
        assert response.status == 200, f"Expected ready to return 200, got {response.status}"

    # Step 4: Put i1 into offline state to test latching behavior
    i1.call("pico._inject_error", "SENTINEL_CONNECTION_POOL_CALL_FAILURE", True)

    assert i1.cluster_name is not None
    assert i1.cluster_uuid is not None
    leader = cluster.leader()
    leader.call(
        ".proc_update_instance_v2",
        [
            i1.name,
            i1.cluster_name,
            i1.cluster_uuid,
            None,  # current_state
            "Offline",  # target_state
            None,  # failure_domain
            False,  # dont_retry
            None,  # picodata_version
        ],
        "k8s_probes_test",
    )
    cluster.wait_has_states(i1, "Offline", "Offline")

    with get_unauthorized(f"{base_url}/startup") as response:
        assert response.status == 200, "Startup should remain 200 after going offline (latched)"

    with get_unauthorized(f"{base_url}/live") as response:
        assert response.status == 200, "Live should return 200 even when offline"

    try:
        with get_unauthorized(f"{base_url}/ready") as response:
            assert False, f"Expected ready to return 503 when offline, got {response.status}"
    except HTTPError as e:
        assert e.code == 503, f"Expected 503, got {e.code}"
        body = json.loads(e.read())
        assert body["status"] == "not_ready"
        assert "offline" in body["reason"].lower()

    # Cleanup
    i1.call("pico._inject_error", "SENTINEL_CONNECTION_POOL_CALL_FAILURE", False)


def test_healthcheck_status_api(cluster: Cluster):
    """
    Test health check status API (/api/v1/health/status) lifecycle.

    This test validates:
    1. Healthy state with data accuracy (instance info, raft, buckets, cluster)
    2. Degraded state when limbo_owner != 0
    3. Unhealthy state when instance is offline (includes degraded reasons)
    """
    i1, i2, i3 = cluster.deploy(instance_count=3, enable_http=True)
    http_listen = i1.http_listen
    status_url = f"http://{http_listen}/api/v1/health/status"

    create_user(i1)
    auth_token = get_auth_token(i1)

    # Wait for startup and bucket distribution
    Retriable().call(lambda: get_authorized(f"http://{http_listen}/api/v1/health/startup", auth_token))
    cluster.wait_until_buckets_balanced()

    # Validate healthy status with data accuracy
    with get_authorized(status_url, auth_token) as response:
        assert response.status == 200, "Status endpoint should return 200"
        body = json.load(response)

    # Get actual instance info for comparison
    instance_info = i1.call(".proc_instance_info")
    raft_info = i1.call(".proc_raft_info")

    # Instance info
    assert body["name"] == i1.name
    assert body["uuid"] == instance_info["uuid"]
    assert body["raftId"] == instance_info["raft_id"]
    assert body["tier"] == instance_info["tier"]
    assert body["replicaset"] == instance_info["replicaset_name"]
    assert body["currentState"] == instance_info["current_state"]["variant"]
    assert body["targetState"] == instance_info["target_state"]["variant"]
    assert body["version"] == i1.picodata_version()

    # Raft status
    raft = body["raft"]
    assert raft["term"] == raft_info["term"]
    assert raft["leaderId"] == raft_info["leader_id"]
    assert raft["appliedIndex"] == raft_info["applied"]

    # Bucket status
    buckets = body["buckets"]
    tier_name = instance_info["tier"]
    expected_bucket_count = i1.sql(f"SELECT bucket_count FROM _pico_tier WHERE name = '{tier_name}'")[0][0]
    assert buckets["total"] == expected_bucket_count
    assert buckets["active"] > 0
    assert buckets["sending"] == 0
    assert buckets["garbage"] == 0

    # Cluster info
    cluster_info = body["cluster"]
    assert cluster_info["uuid"] == i1.cluster_uuid
    expected_version = i1.call("box.space._pico_property:get", "cluster_version")[1]
    assert cluster_info["version"] == expected_version

    # Timestamp and uptime
    assert body["timestamp"] > 0
    assert body["uptimeSeconds"] > 0

    # Healthy status
    assert body["status"] == "healthy", f"Expected healthy, got: {body['status']}"
    assert len(body.get("issues", [])) == 0, "Expected no issues for healthy status"

    # Validate degraded status (limbo_owner != 0)
    i1.eval("box.ctl.promote()")
    time.sleep(0.2)

    limbo_owner = i1.eval("return box.info.synchro.queue.owner")
    assert limbo_owner != 0, "box.ctl.promote() should set limbo_owner"

    with get_authorized(status_url, auth_token) as response:
        assert response.status == 200
        body = json.load(response)

    assert body["status"] == "degraded", f"Expected degraded, got: {body['status']}"
    assert body["limboOwner"] == limbo_owner, f"Expected limboOwner={limbo_owner}, got: {body['limboOwner']}"
    issues = body.get("issues", [])
    assert any("limbo" in r.lower() for r in issues), f"Expected limbo issue: {issues}"

    # Validate unhealthy status
    # inject SENTINEL_CONNECTION_POOL_CALL_FAILURE to disable instance auto-online
    i1.call("pico._inject_error", "SENTINEL_CONNECTION_POOL_CALL_FAILURE", True)

    assert i1.cluster_name is not None
    assert i1.cluster_uuid is not None
    leader = cluster.leader()
    leader.call(
        ".proc_update_instance_v2",
        [
            i1.name,
            i1.cluster_name,
            i1.cluster_uuid,
            None,
            "Offline",
            None,
            False,
            None,
        ],
        "status_api_test",
    )
    cluster.wait_has_states(i1, "Offline", "Offline")

    with get_authorized(status_url, auth_token) as response:
        assert response.status == 200
        body = json.load(response)

    # Broken overrides degraded
    assert body["status"] == "broken", f"Expected broken, got: {body['status']}"
    assert body["targetStateReason"] == "status_api_test"
    issues = body.get("issues", [])
    # Should include both broken and degraded issues
    assert any("offline" in r.lower() for r in issues), f"Expected offline issue: {issues}"
    assert any("limbo" in r.lower() for r in issues), f"Expected limbo issue: {issues}"

    # Cleanup
    i1.call("pico._inject_error", "SENTINEL_CONNECTION_POOL_CALL_FAILURE", False)


def test_kubernetes_probes_disabled(cluster: Cluster):
    """
    Test that kubernetes_probes: false disables K8s health probe endpoints.
    The endpoints should return 404 when disabled.
    """
    cluster.set_config_file(
        yaml=f"""
cluster:
    name: {cluster.id}
    tier:
        default:
            replication_factor: 1
instance:
    http:
        kubernetes_probes: false
"""
    )

    i1 = cluster.add_instance(wait_online=True, tier="default", enable_http=True)
    http_listen = i1.http_listen
    base_url = f"http://{http_listen}/api/v1/health"

    # K8s probes should return 404 when disabled
    for endpoint in ["live", "ready", "startup"]:
        try:
            with get_unauthorized(f"{base_url}/{endpoint}") as response:
                assert False, f"Expected 404 for {endpoint}, got {response.status}"
        except HTTPError as e:
            assert e.code == 404, f"Expected 404 for {endpoint}, got {e.code}"
