Skip to content

Commit 8fdb45d

Browse files
seratchseratch
authored
fix: update default reasoning effort for newer models (#2773)
1 parent abd5cef commit 8fdb45d

2 files changed

Lines changed: 147 additions & 60 deletions

File tree

‎src/agents/models/default_models.py‎

Lines changed: 56 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,19 @@
11
import copy
22
import os
3-
from typing import Optional
3+
import re
4+
from typing import Literal, Optional
45

56
from openai.types.shared.reasoning import Reasoning
67

78
from agents.model_settings import ModelSettings
89

910
OPENAI_DEFAULT_MODEL_ENV_VARIABLE_NAME = "OPENAI_DEFAULT_MODEL"
1011

11-
# discourage directly accessing this constant
12+
GPT5DefaultReasoningEffort = Literal["none", "low", "medium"]
13+
14+
# discourage directly accessing these constants
1215
# use the get_default_model and get_default_model_settings() functions instead
13-
_GPT_5_DEFAULT_MODEL_SETTINGS: ModelSettings = ModelSettings(
16+
_GPT_5_LOW_DEFAULT_MODEL_SETTINGS: ModelSettings = ModelSettings(
1417
# We chose "low" instead of "minimal" because some of the built-in tools
1518
# (e.g., file search, image generation, etc.) do not support "minimal"
1619
# If you want to use "minimal" reasoning effort, you can pass your own model settings
@@ -21,20 +24,59 @@
2124
reasoning=Reasoning(effort="none"),
2225
verbosity="low",
2326
)
27+
_GPT_5_MEDIUM_DEFAULT_MODEL_SETTINGS: ModelSettings = ModelSettings(
28+
reasoning=Reasoning(effort="medium"),
29+
verbosity="low",
30+
)
31+
_GPT_5_TEXT_ONLY_DEFAULT_MODEL_SETTINGS: ModelSettings = ModelSettings(
32+
verbosity="low",
33+
)
2434

25-
_GPT_5_NONE_EFFORT_MODELS = {"gpt-5.1", "gpt-5.2"}
35+
_GPT_5_CHAT_MODEL_PATTERNS: tuple[re.Pattern[str], ...] = (
36+
re.compile(r"^gpt-5-chat-latest$"),
37+
re.compile(r"^gpt-5\.1-chat-latest$"),
38+
re.compile(r"^gpt-5\.2-chat-latest$"),
39+
re.compile(r"^gpt-5\.3-chat-latest$"),
40+
)
41+
42+
_GPT_5_DEFAULT_MODEL_SETTINGS_BY_REASONING_EFFORT: dict[
43+
GPT5DefaultReasoningEffort, ModelSettings
44+
] = {
45+
"none": _GPT_5_NONE_DEFAULT_MODEL_SETTINGS,
46+
"low": _GPT_5_LOW_DEFAULT_MODEL_SETTINGS,
47+
"medium": _GPT_5_MEDIUM_DEFAULT_MODEL_SETTINGS,
48+
}
49+
50+
_GPT_5_DEFAULT_REASONING_EFFORT_PATTERNS: tuple[
51+
tuple[re.Pattern[str], GPT5DefaultReasoningEffort],
52+
...,
53+
] = (
54+
(re.compile(r"^gpt-5(?:-\d{4}-\d{2}-\d{2})?$"), "low"),
55+
(re.compile(r"^gpt-5\.1(?:-\d{4}-\d{2}-\d{2})?$"), "none"),
56+
(re.compile(r"^gpt-5\.2(?:-\d{4}-\d{2}-\d{2})?$"), "none"),
57+
(re.compile(r"^gpt-5\.2-pro(?:-\d{4}-\d{2}-\d{2})?$"), "medium"),
58+
(re.compile(r"^gpt-5\.2-codex$"), "low"),
59+
(re.compile(r"^gpt-5\.3-codex$"), "none"),
60+
(re.compile(r"^gpt-5\.4(?:-\d{4}-\d{2}-\d{2})?$"), "none"),
61+
(re.compile(r"^gpt-5\.4-pro(?:-\d{4}-\d{2}-\d{2})?$"), "medium"),
62+
(re.compile(r"^gpt-5\.4-mini(?:-\d{4}-\d{2}-\d{2})?$"), "none"),
63+
(re.compile(r"^gpt-5\.4-nano(?:-\d{4}-\d{2}-\d{2})?$"), "none"),
64+
)
2665

2766

28-
def _is_gpt_5_none_effort_model(model_name: str) -> bool:
29-
return model_name in _GPT_5_NONE_EFFORT_MODELS
67+
def _get_default_reasoning_effort(model_name: str) -> GPT5DefaultReasoningEffort | None:
68+
for pattern, effort in _GPT_5_DEFAULT_REASONING_EFFORT_PATTERNS:
69+
if pattern.fullmatch(model_name):
70+
return effort
71+
return None
3072

3173

3274
def gpt_5_reasoning_settings_required(model_name: str) -> bool:
3375
"""
3476
Returns True if the model name is a GPT-5 model and reasoning settings are required.
3577
"""
36-
if model_name.startswith("gpt-5-chat"):
37-
# gpt-5-chat-latest does not require reasoning settings
78+
if any(pattern.fullmatch(model_name) for pattern in _GPT_5_CHAT_MODEL_PATTERNS):
79+
# Chat-latest aliases do not accept reasoning.effort.
3880
return False
3981
# matches any of gpt-5 models
4082
return model_name.startswith("gpt-5")
@@ -64,7 +106,10 @@ def get_default_model_settings(model: Optional[str] = None) -> ModelSettings:
64106
"""
65107
_model = model if model is not None else get_default_model()
66108
if gpt_5_reasoning_settings_required(_model):
67-
if _is_gpt_5_none_effort_model(_model):
68-
return copy.deepcopy(_GPT_5_NONE_DEFAULT_MODEL_SETTINGS)
69-
return copy.deepcopy(_GPT_5_DEFAULT_MODEL_SETTINGS)
109+
effort = _get_default_reasoning_effort(_model)
110+
if effort is not None:
111+
return copy.deepcopy(_GPT_5_DEFAULT_MODEL_SETTINGS_BY_REASONING_EFFORT[effort])
112+
# Keep the GPT-5 verbosity default, but omit reasoning.effort for
113+
# variants whose supported values are not confirmed yet.
114+
return copy.deepcopy(_GPT_5_TEXT_ONLY_DEFAULT_MODEL_SETTINGS)
70115
return ModelSettings()

‎tests/models/test_default_models.py‎

Lines changed: 91 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
import os
2+
from typing import Literal
23
from unittest.mock import patch
34

5+
from openai.types.shared.reasoning import Reasoning
6+
47
from agents import Agent
58
from agents.model_settings import ModelSettings
69
from agents.models import (
@@ -11,75 +14,114 @@
1114
)
1215

1316

17+
def _gpt_5_default_settings(
18+
reasoning_effort: Literal["none", "low", "medium"] | None,
19+
) -> ModelSettings:
20+
if reasoning_effort is None:
21+
return ModelSettings(verbosity="low")
22+
return ModelSettings(reasoning=Reasoning(effort=reasoning_effort), verbosity="low")
23+
24+
1425
def test_default_model_is_gpt_4_1():
1526
assert get_default_model() == "gpt-4.1"
1627
assert is_gpt_5_default() is False
1728
assert gpt_5_reasoning_settings_required(get_default_model()) is False
1829
assert get_default_model_settings().reasoning is None
1930

2031

21-
@patch.dict(os.environ, {"OPENAI_DEFAULT_MODEL": "gpt-5"})
22-
def test_default_model_env_gpt_5():
23-
assert get_default_model() == "gpt-5"
32+
@patch.dict(os.environ, {"OPENAI_DEFAULT_MODEL": "gpt-5.4"})
33+
def test_is_gpt_5_default_with_real_model_name():
34+
assert get_default_model() == "gpt-5.4"
2435
assert is_gpt_5_default() is True
25-
assert gpt_5_reasoning_settings_required(get_default_model()) is True
26-
assert get_default_model_settings().reasoning.effort == "low" # type: ignore[union-attr]
2736

2837

29-
@patch.dict(os.environ, {"OPENAI_DEFAULT_MODEL": "gpt-5.1"})
30-
def test_default_model_env_gpt_5_1():
31-
assert get_default_model() == "gpt-5.1"
32-
assert is_gpt_5_default() is True
33-
assert gpt_5_reasoning_settings_required(get_default_model()) is True
34-
assert get_default_model_settings().reasoning.effort == "none" # type: ignore[union-attr]
38+
@patch.dict(os.environ, {"OPENAI_DEFAULT_MODEL": "gpt-4.1"})
39+
def test_is_gpt_5_default_returns_false_for_non_gpt_5_default_model():
40+
assert get_default_model() == "gpt-4.1"
41+
assert is_gpt_5_default() is False
3542

3643

37-
@patch.dict(os.environ, {"OPENAI_DEFAULT_MODEL": "gpt-5.2"})
38-
def test_default_model_env_gpt_5_2():
39-
assert get_default_model() == "gpt-5.2"
40-
assert is_gpt_5_default() is True
41-
assert gpt_5_reasoning_settings_required(get_default_model()) is True
42-
assert get_default_model_settings().reasoning.effort == "none" # type: ignore[union-attr]
44+
def test_gpt_5_reasoning_settings_required_detects_gpt_5_models_while_ignoring_chat_latest():
45+
assert gpt_5_reasoning_settings_required("gpt-5") is True
46+
assert gpt_5_reasoning_settings_required("gpt-5.1") is True
47+
assert gpt_5_reasoning_settings_required("gpt-5.2") is True
48+
assert gpt_5_reasoning_settings_required("gpt-5.2-codex") is True
49+
assert gpt_5_reasoning_settings_required("gpt-5.2-pro") is True
50+
assert gpt_5_reasoning_settings_required("gpt-5.4-pro") is True
51+
assert gpt_5_reasoning_settings_required("gpt-5-mini") is True
52+
assert gpt_5_reasoning_settings_required("gpt-5-nano") is True
53+
assert gpt_5_reasoning_settings_required("gpt-5-chat-latest") is False
54+
assert gpt_5_reasoning_settings_required("gpt-5.1-chat-latest") is False
55+
assert gpt_5_reasoning_settings_required("gpt-5.2-chat-latest") is False
56+
assert gpt_5_reasoning_settings_required("gpt-5.3-chat-latest") is False
4357

4458

45-
@patch.dict(os.environ, {"OPENAI_DEFAULT_MODEL": "gpt-5.2-codex"})
46-
def test_default_model_env_gpt_5_2_codex():
47-
assert get_default_model() == "gpt-5.2-codex"
48-
assert is_gpt_5_default() is True
49-
assert gpt_5_reasoning_settings_required(get_default_model()) is True
50-
assert get_default_model_settings().reasoning.effort == "low" # type: ignore[union-attr]
59+
def test_gpt_5_reasoning_settings_required_returns_false_for_non_gpt_5_models():
60+
assert gpt_5_reasoning_settings_required("gpt-4.1") is False
5161

5262

53-
@patch.dict(os.environ, {"OPENAI_DEFAULT_MODEL": "gpt-5-mini"})
54-
def test_default_model_env_gpt_5_mini():
55-
assert get_default_model() == "gpt-5-mini"
56-
assert is_gpt_5_default() is True
57-
assert gpt_5_reasoning_settings_required(get_default_model()) is True
58-
assert get_default_model_settings().reasoning.effort == "low" # type: ignore[union-attr]
63+
def test_get_default_model_settings_returns_none_reasoning_defaults_for_gpt_5_1_models():
64+
assert get_default_model_settings("gpt-5.1") == _gpt_5_default_settings("none")
65+
assert get_default_model_settings("gpt-5.1-2025-11-13") == _gpt_5_default_settings("none")
5966

6067

61-
@patch.dict(os.environ, {"OPENAI_DEFAULT_MODEL": "gpt-5-nano"})
62-
def test_default_model_env_gpt_5_nano():
63-
assert get_default_model() == "gpt-5-nano"
64-
assert is_gpt_5_default() is True
65-
assert gpt_5_reasoning_settings_required(get_default_model()) is True
66-
assert get_default_model_settings().reasoning.effort == "low" # type: ignore[union-attr]
68+
def test_get_default_model_settings_returns_none_reasoning_defaults_for_gpt_5_2_models():
69+
assert get_default_model_settings("gpt-5.2") == _gpt_5_default_settings("none")
70+
assert get_default_model_settings("gpt-5.2-2025-12-11") == _gpt_5_default_settings("none")
6771

6872

69-
@patch.dict(os.environ, {"OPENAI_DEFAULT_MODEL": "gpt-5-chat-latest"})
70-
def test_default_model_env_gpt_5_chat_latest():
71-
assert get_default_model() == "gpt-5-chat-latest"
72-
assert is_gpt_5_default() is False
73-
assert gpt_5_reasoning_settings_required(get_default_model()) is False
74-
assert get_default_model_settings().reasoning is None
73+
def test_get_default_model_settings_returns_none_reasoning_defaults_for_gpt_5_3_codex_models():
74+
assert get_default_model_settings("gpt-5.3-codex") == _gpt_5_default_settings("none")
7575

7676

77-
@patch.dict(os.environ, {"OPENAI_DEFAULT_MODEL": "gpt-4o"})
78-
def test_default_model_env_gpt_4o():
79-
assert get_default_model() == "gpt-4o"
80-
assert is_gpt_5_default() is False
81-
assert gpt_5_reasoning_settings_required(get_default_model()) is False
82-
assert get_default_model_settings().reasoning is None
77+
def test_get_default_model_settings_returns_none_reasoning_defaults_for_gpt_5_4_models():
78+
assert get_default_model_settings("gpt-5.4") == _gpt_5_default_settings("none")
79+
80+
81+
def test_get_default_model_settings_returns_none_reasoning_defaults_for_gpt_5_4_snapshot_families():
82+
assert get_default_model_settings("gpt-5.4-2026-03-05") == _gpt_5_default_settings("none")
83+
assert get_default_model_settings("gpt-5.4-mini-2026-03-17") == _gpt_5_default_settings("none")
84+
assert get_default_model_settings("gpt-5.4-nano-2026-03-17") == _gpt_5_default_settings("none")
85+
86+
87+
def test_get_default_model_settings_returns_none_reasoning_defaults_for_gpt_5_4_mini_and_nano():
88+
assert get_default_model_settings("gpt-5.4-mini") == _gpt_5_default_settings("none")
89+
assert get_default_model_settings("gpt-5.4-nano") == _gpt_5_default_settings("none")
90+
91+
92+
def test_get_default_model_settings_returns_low_reasoning_defaults_for_base_gpt_5():
93+
assert get_default_model_settings("gpt-5") == _gpt_5_default_settings("low")
94+
assert get_default_model_settings("gpt-5-2025-08-07") == _gpt_5_default_settings("low")
95+
96+
97+
def test_get_default_model_settings_returns_low_reasoning_defaults_for_gpt_5_2_codex():
98+
assert get_default_model_settings("gpt-5.2-codex") == _gpt_5_default_settings("low")
99+
100+
101+
def test_get_default_model_settings_returns_medium_reasoning_defaults_for_gpt_5_pro_models():
102+
assert get_default_model_settings("gpt-5.2-pro") == _gpt_5_default_settings("medium")
103+
assert get_default_model_settings("gpt-5.2-pro-2025-12-11") == _gpt_5_default_settings("medium")
104+
assert get_default_model_settings("gpt-5.4-pro") == _gpt_5_default_settings("medium")
105+
assert get_default_model_settings("gpt-5.4-pro-2026-03-05") == _gpt_5_default_settings("medium")
106+
107+
108+
def test_get_default_model_settings_omits_reasoning_for_unconfirmed_gpt_5_variants():
109+
assert get_default_model_settings("gpt-5-mini") == _gpt_5_default_settings(None)
110+
assert get_default_model_settings("gpt-5-mini-2025-08-07") == _gpt_5_default_settings(None)
111+
assert get_default_model_settings("gpt-5-nano") == _gpt_5_default_settings(None)
112+
assert get_default_model_settings("gpt-5-nano-2025-08-07") == _gpt_5_default_settings(None)
113+
assert get_default_model_settings("gpt-5.1-codex") == _gpt_5_default_settings(None)
114+
115+
116+
def test_get_default_model_settings_returns_empty_settings_for_gpt_5_chat_latest_aliases():
117+
assert get_default_model_settings("gpt-5-chat-latest") == ModelSettings()
118+
assert get_default_model_settings("gpt-5.1-chat-latest") == ModelSettings()
119+
assert get_default_model_settings("gpt-5.2-chat-latest") == ModelSettings()
120+
assert get_default_model_settings("gpt-5.3-chat-latest") == ModelSettings()
121+
122+
123+
def test_get_default_model_settings_returns_empty_settings_for_non_gpt_5_models():
124+
assert get_default_model_settings("gpt-4.1") == ModelSettings()
83125

84126

85127
@patch.dict(os.environ, {"OPENAI_DEFAULT_MODEL": "gpt-5"})
@@ -94,6 +136,6 @@ def test_agent_uses_gpt_5_default_model_settings():
94136
@patch.dict(os.environ, {"OPENAI_DEFAULT_MODEL": "gpt-5"})
95137
def test_agent_resets_model_settings_for_non_gpt_5_models():
96138
"""Agent should reset default GPT-5 settings when using a non-GPT-5 model."""
97-
agent = Agent(name="test", model="gpt-4o")
98-
assert agent.model == "gpt-4o"
139+
agent = Agent(name="test", model="gpt-4.1")
140+
assert agent.model == "gpt-4.1"
99141
assert agent.model_settings == ModelSettings()

0 commit comments

Comments
 (0)