vibe/tests/core/experiments/test_session_helpers.py

from __future__ import annotations

from typing import Any
from unittest.mock import AsyncMock, MagicMock

import pytest

from vibe.core.experiments.client import RemoteEvalClient
from vibe.core.experiments.manager import ExperimentManager
from vibe.core.experiments.models import EvalResponse, ExperimentAttributes
from vibe.core.experiments.session import (
    hydrate_experiments_from_session,
    initialize_experiments,
)


class _StubClient(RemoteEvalClient):
    def __init__(self, response: EvalResponse | None) -> None:
        self._response = response

    async def evaluate(self, attributes: ExperimentAttributes) -> EvalResponse | None:
        return self._response

    async def aclose(self) -> None:
        pass


def _make_config(
    *, enable_telemetry: bool = True, enable_experiments: bool = True
) -> Any:
    config = MagicMock()
    config.enable_telemetry = enable_telemetry
    config.experiments.enable = enable_experiments
    return config


@pytest.mark.asyncio
async def test_initialize_returns_false_when_telemetry_disabled(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    persist = AsyncMock()
    session_logger = MagicMock()
    session_logger.persist_experiments = persist
    manager = ExperimentManager(client=_StubClient(None))

    result = await initialize_experiments(
        config=_make_config(enable_telemetry=False),
        manager=manager,
        session_logger=session_logger,
        entrypoint_metadata=None,
    )

    assert result is False
    persist.assert_not_called()


@pytest.mark.asyncio
async def test_initialize_returns_false_when_experiments_disabled(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    # `experiments.enable=False` must short-circuit even when telemetry is on
    # — it is the user's opt-out for being assigned to A/B tests without
    # disabling all telemetry.
    persist = AsyncMock()
    session_logger = MagicMock()
    session_logger.persist_experiments = persist
    manager = ExperimentManager(client=_StubClient(None))

    result = await initialize_experiments(
        config=_make_config(enable_experiments=False),
        manager=manager,
        session_logger=session_logger,
        entrypoint_metadata=None,
    )

    assert result is False
    persist.assert_not_called()


@pytest.mark.asyncio
async def test_initialize_returns_false_when_no_mistral_provider(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    monkeypatch.setattr(
        "vibe.core.experiments.session.get_mistral_provider_and_api_key",
        lambda _config: None,
    )
    persist = AsyncMock()
    session_logger = MagicMock()
    session_logger.persist_experiments = persist
    manager = ExperimentManager(client=_StubClient(None))

    result = await initialize_experiments(
        config=_make_config(),
        manager=manager,
        session_logger=session_logger,
        entrypoint_metadata=None,
    )

    assert result is False
    persist.assert_not_called()


@pytest.mark.asyncio
async def test_initialize_returns_false_when_remote_eval_fails(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    # Regression: even if telemetry is enabled and a Mistral key is set,
    # a failed remote eval (returns None) leaves manager state empty —
    # the helper must NOT report success, so the caller skips the
    # unnecessary system prompt refresh.
    monkeypatch.setattr(
        "vibe.core.experiments.session.get_mistral_provider_and_api_key",
        lambda _config: (MagicMock(), "fake-key"),
    )
    monkeypatch.setattr(
        "vibe.core.experiments.session._build_attributes",
        lambda *_args, **_kwargs: ExperimentAttributes(
            userId="x", entrypoint="cli", agent_version="0", os="darwin"
        ),
    )
    persist = AsyncMock()
    session_logger = MagicMock()
    session_logger.persist_experiments = persist
    manager = ExperimentManager(client=_StubClient(None))

    result = await initialize_experiments(
        config=_make_config(),
        manager=manager,
        session_logger=session_logger,
        entrypoint_metadata=None,
    )

    assert result is False
    persist.assert_not_called()


@pytest.mark.asyncio
async def test_initialize_returns_true_and_persists_when_remote_eval_succeeds(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    monkeypatch.setattr(
        "vibe.core.experiments.session.get_mistral_provider_and_api_key",
        lambda _config: (MagicMock(), "fake-key"),
    )
    monkeypatch.setattr(
        "vibe.core.experiments.session._build_attributes",
        lambda *_args, **_kwargs: ExperimentAttributes(
            userId="x", entrypoint="cli", agent_version="0", os="darwin"
        ),
    )
    persist = AsyncMock()
    session_logger = MagicMock()
    session_logger.persist_experiments = persist
    response = EvalResponse.model_validate({
        "features": {"vibe_cli_system_prompt": {"defaultValue": "cli"}}
    })
    manager = ExperimentManager(client=_StubClient(response))

    result = await initialize_experiments(
        config=_make_config(),
        manager=manager,
        session_logger=session_logger,
        entrypoint_metadata=None,
    )

    assert result is True
    persist.assert_awaited_once()


@pytest.mark.asyncio
async def test_hydrate_returns_false_when_telemetry_disabled() -> None:
    session_logger = MagicMock()
    response = EvalResponse.model_validate({
        "features": {"vibe_cli_system_prompt": {"defaultValue": "cli"}}
    })
    session_logger.session_metadata.experiments = response
    manager = ExperimentManager(client=_StubClient(None))

    result = await hydrate_experiments_from_session(
        config=_make_config(enable_telemetry=False),
        manager=manager,
        session_logger=session_logger,
    )

    assert result is False
    assert manager.export_state() is None


@pytest.mark.asyncio
async def test_hydrate_returns_false_when_experiments_disabled() -> None:
    # Regression: without this gate, a user who flipped experiments.enable
    # to False between sessions would still resume into a hydrated variant.
    session_logger = MagicMock()
    response = EvalResponse.model_validate({
        "features": {"vibe_cli_system_prompt": {"defaultValue": "cli"}}
    })
    session_logger.session_metadata.experiments = response
    manager = ExperimentManager(client=_StubClient(None))

    result = await hydrate_experiments_from_session(
        config=_make_config(enable_experiments=False),
        manager=manager,
        session_logger=session_logger,
    )

    assert result is False
    assert manager.export_state() is None