v2.10.0 (#697)

Co-authored-by: Clément Drouin <clement.drouin@mistral.ai>
Co-authored-by: Corentin André <corentin.andre@mistral.ai>
Co-authored-by: Guillaume LE GOFF <guillaume.lgf@gmail.com>
Co-authored-by: Kim-Adeline Miguel <51720070+kimadeline@users.noreply.github.com>
Co-authored-by: Maxime Dolores <maxime.dolores@ext.mistral.ai>
Co-authored-by: Nelson PROIA <144663685+Nelson-PROIA@users.noreply.github.com>
Co-authored-by: Peter Evers <pevers90@gmail.com>
Co-authored-by: Pierre Rossinès <pierre.rossines@mistral.ai>
Co-authored-by: Quentin <quentin.torroba@mistral.ai>
Co-authored-by: Vincent G <10739306+VinceOPS@users.noreply.github.com>
Co-authored-by: MichisGitIsKing <MichisGitIsKing@users.noreply.github.com>
Co-authored-by: Mistral Vibe <vibe@mistral.ai>

tests/core/experiments/test_session_helpers.py

@@ -0,0 +1,208 @@
+from __future__ import annotations
+
+from typing import Any
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+from vibe.core.experiments.client import RemoteEvalClient
+from vibe.core.experiments.manager import ExperimentManager
+from vibe.core.experiments.models import EvalResponse, ExperimentAttributes
+from vibe.core.experiments.session import (
+    hydrate_experiments_from_session,
+    initialize_experiments,
+)
+
+
+class _StubClient(RemoteEvalClient):
+    def __init__(self, response: EvalResponse | None) -> None:
+        self._response = response
+
+    async def evaluate(self, attributes: ExperimentAttributes) -> EvalResponse | None:
+        return self._response
+
+    async def aclose(self) -> None:
+        pass
+
+
+def _make_config(
+    *, enable_telemetry: bool = True, enable_experiments: bool = True
+) -> Any:
+    config = MagicMock()
+    config.enable_telemetry = enable_telemetry
+    config.experiments.enable = enable_experiments
+    return config
+
+
+@pytest.mark.asyncio
+async def test_initialize_returns_false_when_telemetry_disabled(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    persist = AsyncMock()
+    session_logger = MagicMock()
+    session_logger.persist_experiments = persist
+    manager = ExperimentManager(client=_StubClient(None))
+
+    result = await initialize_experiments(
+        config=_make_config(enable_telemetry=False),
+        manager=manager,
+        session_logger=session_logger,
+        entrypoint_metadata=None,
+    )
+
+    assert result is False
+    persist.assert_not_called()
+
+
+@pytest.mark.asyncio
+async def test_initialize_returns_false_when_experiments_disabled(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    # `experiments.enable=False` must short-circuit even when telemetry is on
+    # — it is the user's opt-out for being assigned to A/B tests without
+    # disabling all telemetry.
+    persist = AsyncMock()
+    session_logger = MagicMock()
+    session_logger.persist_experiments = persist
+    manager = ExperimentManager(client=_StubClient(None))
+
+    result = await initialize_experiments(
+        config=_make_config(enable_experiments=False),
+        manager=manager,
+        session_logger=session_logger,
+        entrypoint_metadata=None,
+    )
+
+    assert result is False
+    persist.assert_not_called()
+
+
+@pytest.mark.asyncio
+async def test_initialize_returns_false_when_no_mistral_provider(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setattr(
+        "vibe.core.experiments.session.get_mistral_provider_and_api_key",
+        lambda _config: None,
+    )
+    persist = AsyncMock()
+    session_logger = MagicMock()
+    session_logger.persist_experiments = persist
+    manager = ExperimentManager(client=_StubClient(None))
+
+    result = await initialize_experiments(
+        config=_make_config(),
+        manager=manager,
+        session_logger=session_logger,
+        entrypoint_metadata=None,
+    )
+
+    assert result is False
+    persist.assert_not_called()
+
+
+@pytest.mark.asyncio
+async def test_initialize_returns_false_when_remote_eval_fails(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    # Regression: even if telemetry is enabled and a Mistral key is set,
+    # a failed remote eval (returns None) leaves manager state empty —
+    # the helper must NOT report success, so the caller skips the
+    # unnecessary system prompt refresh.
+    monkeypatch.setattr(
+        "vibe.core.experiments.session.get_mistral_provider_and_api_key",
+        lambda _config: (MagicMock(), "fake-key"),
+    )
+    monkeypatch.setattr(
+        "vibe.core.experiments.session._build_attributes",
+        lambda *_args, **_kwargs: ExperimentAttributes(
+            userId="x", entrypoint="cli", agent_version="0", os="darwin"
+        ),
+    )
+    persist = AsyncMock()
+    session_logger = MagicMock()
+    session_logger.persist_experiments = persist
+    manager = ExperimentManager(client=_StubClient(None))
+
+    result = await initialize_experiments(
+        config=_make_config(),
+        manager=manager,
+        session_logger=session_logger,
+        entrypoint_metadata=None,
+    )
+
+    assert result is False
+    persist.assert_not_called()
+
+
+@pytest.mark.asyncio
+async def test_initialize_returns_true_and_persists_when_remote_eval_succeeds(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setattr(
+        "vibe.core.experiments.session.get_mistral_provider_and_api_key",
+        lambda _config: (MagicMock(), "fake-key"),
+    )
+    monkeypatch.setattr(
+        "vibe.core.experiments.session._build_attributes",
+        lambda *_args, **_kwargs: ExperimentAttributes(
+            userId="x", entrypoint="cli", agent_version="0", os="darwin"
+        ),
+    )
+    persist = AsyncMock()
+    session_logger = MagicMock()
+    session_logger.persist_experiments = persist
+    response = EvalResponse.model_validate({
+        "features": {"vibe_cli_system_prompt": {"defaultValue": "cli"}}
+    })
+    manager = ExperimentManager(client=_StubClient(response))
+
+    result = await initialize_experiments(
+        config=_make_config(),
+        manager=manager,
+        session_logger=session_logger,
+        entrypoint_metadata=None,
+    )
+
+    assert result is True
+    persist.assert_awaited_once()
+
+
+@pytest.mark.asyncio
+async def test_hydrate_returns_false_when_telemetry_disabled() -> None:
+    session_logger = MagicMock()
+    response = EvalResponse.model_validate({
+        "features": {"vibe_cli_system_prompt": {"defaultValue": "cli"}}
+    })
+    session_logger.session_metadata.experiments = response
+    manager = ExperimentManager(client=_StubClient(None))
+
+    result = await hydrate_experiments_from_session(
+        config=_make_config(enable_telemetry=False),
+        manager=manager,
+        session_logger=session_logger,
+    )
+
+    assert result is False
+    assert manager.export_state() is None
+
+
+@pytest.mark.asyncio
+async def test_hydrate_returns_false_when_experiments_disabled() -> None:
+    # Regression: without this gate, a user who flipped experiments.enable
+    # to False between sessions would still resume into a hydrated variant.
+    session_logger = MagicMock()
+    response = EvalResponse.model_validate({
+        "features": {"vibe_cli_system_prompt": {"defaultValue": "cli"}}
+    })
+    session_logger.session_metadata.experiments = response
+    manager = ExperimentManager(client=_StubClient(None))
+
+    result = await hydrate_experiments_from_session(
+        config=_make_config(enable_experiments=False),
+        manager=manager,
+        session_logger=session_logger,
+    )
+
+    assert result is False
+    assert manager.export_state() is None