diff --git a/sagemaker-train/src/sagemaker/train/constants.py b/sagemaker-train/src/sagemaker/train/constants.py index 212a57e642..2f11d4d49e 100644 --- a/sagemaker-train/src/sagemaker/train/constants.py +++ b/sagemaker-train/src/sagemaker/train/constants.py @@ -58,15 +58,25 @@ def get_sagemaker_hub_name() -> str: "qwen.qwen3-235b-a22b-2507-v1:0": ["us-west-2", "ap-northeast-1"] } -# Allowed evaluator models for LLM as Judge evaluator with region restrictions +# Allowed evaluator models for LLM as Judge evaluator with region restrictions. +# +# Source of truth: the Bedrock Console judge-model regional +# allowlist.cross-checked against +# https://docs.aws.amazon.com/bedrock/latest/userguide/evaluation-judge.html#evaluation-judge-supported _ALLOWED_EVALUATOR_MODELS = { - "anthropic.claude-3-5-sonnet-20240620-v1:0": ["us-west-2", "us-east-1", "ap-northeast-1"], - "anthropic.claude-3-5-sonnet-20241022-v2:0": ["us-west-2"], - "anthropic.claude-3-haiku-20240307-v1:0": ["us-west-2", "us-east-1", "ap-northeast-1", "eu-west-1"], - "anthropic.claude-3-5-haiku-20241022-v1:0": ["us-west-2"], - "meta.llama3-1-70b-instruct-v1:0": ["us-west-2"], "mistral.mistral-large-2402-v1:0": ["us-west-2", "us-east-1", "eu-west-1"], - "amazon.nova-pro-v1:0": ["us-east-1"] + "meta.llama3-1-70b-instruct-v1:0": ["us-west-2", "us-east-1"], + "anthropic.claude-3-haiku-20240307-v1:0": ["us-west-2", "us-east-1", "ap-northeast-1", "eu-west-1"], + "anthropic.claude-sonnet-4-20250514-v1:0": ["us-west-2", "us-east-1", "ap-northeast-1", "eu-west-1"], + "anthropic.claude-haiku-4-5-20251001-v1:0": ["us-west-2", "us-east-1", "ap-northeast-1", "eu-west-1"], + "anthropic.claude-sonnet-4-5-20250929-v1:0": ["us-west-2", "us-east-1", "ap-northeast-1", "eu-west-1"], + "anthropic.claude-opus-4-5-20251101-v1:0": ["us-west-2", "us-east-1", "ap-northeast-1", "eu-west-1"], + "amazon.nova-pro-v1:0": ["us-west-2", "us-east-1", "ap-northeast-1", "eu-west-1"], + "amazon.nova-2-lite-v1:0": ["us-west-2", "us-east-1", "ap-northeast-1", "eu-west-1"], + "amazon.nova-micro-v1:0": ["us-west-2", "us-east-1", "ap-northeast-1", "eu-west-1"], + "amazon.nova-premier-v1:0": ["us-west-2", "us-east-1"], + "anthropic.claude-3-5-sonnet-20240620-v1:0": ["ap-northeast-1"], + "anthropic.claude-3-5-sonnet-20241022-v2:0": ["ap-northeast-1"], } SM_RECIPE = "recipe" diff --git a/sagemaker-train/src/sagemaker/train/evaluate/llm_as_judge_evaluator.py b/sagemaker-train/src/sagemaker/train/evaluate/llm_as_judge_evaluator.py index ccf852468a..e1b7b85bc5 100644 --- a/sagemaker-train/src/sagemaker/train/evaluate/llm_as_judge_evaluator.py +++ b/sagemaker-train/src/sagemaker/train/evaluate/llm_as_judge_evaluator.py @@ -110,7 +110,7 @@ class LLMAsJudgeEvaluator(BaseEvaluator): # Both formats work - with or without 'Builtin.' prefix evaluator = LLMAsJudgeEvaluator( base_model="llama-3-3-70b-instruct", - evaluator_model="anthropic.claude-3-5-sonnet-20240620-v1:0", + evaluator_model="anthropic.claude-sonnet-4-20250514-v1:0", dataset="s3://my-bucket/my-dataset.jsonl", builtin_metrics=["Correctness", "Helpfulness"], # Prefix optional mlflow_resource_arn="arn:aws:sagemaker:us-west-2:123456789012:mlflow-tracking-server/my-server", @@ -144,7 +144,7 @@ class LLMAsJudgeEvaluator(BaseEvaluator): # Example evaluating only custom model (skip base model) evaluator = LLMAsJudgeEvaluator( base_model="llama-3-3-70b-instruct", - evaluator_model="anthropic.claude-3-5-sonnet-20240620-v1:0", + evaluator_model="anthropic.claude-sonnet-4-20250514-v1:0", dataset="s3://my-bucket/my-dataset.jsonl", builtin_metrics=["Correctness"], # Prefix optional evaluate_base_model=False, @@ -776,7 +776,7 @@ def evaluate(self): evaluator = LLMAsJudgeEvaluator( base_model="llama-3-3-70b-instruct", - evaluator_model="anthropic.claude-3-5-sonnet-20240620-v1:0", + evaluator_model="anthropic.claude-sonnet-4-20250514-v1:0", dataset="s3://my-bucket/my-dataset.jsonl", builtin_metrics=["Correctness", "Helpfulness"], s3_output_path="s3://my-bucket/output" diff --git a/sagemaker-train/tests/integ/train/test_llm_as_judge_base_model_fix.py b/sagemaker-train/tests/integ/train/test_llm_as_judge_base_model_fix.py index b395547ea9..7ae585966e 100644 --- a/sagemaker-train/tests/integ/train/test_llm_as_judge_base_model_fix.py +++ b/sagemaker-train/tests/integ/train/test_llm_as_judge_base_model_fix.py @@ -70,7 +70,7 @@ ACCOUNT_ID = "729646638167" TEST_CONFIG = { - "evaluator_model": "anthropic.claude-3-5-haiku-20241022-v1:0", + "evaluator_model": "anthropic.claude-sonnet-4-20250514-v1:0", "dataset_s3_uri": f"s3://sagemaker-{REGION}-{ACCOUNT_ID}/model-customization/eval/gen_qa.jsonl", "builtin_metrics": ["Completeness", "Faithfulness"], "custom_metrics_json": json.dumps([CUSTOM_METRIC_DICT]), diff --git a/sagemaker-train/tests/integ/train/test_llm_as_judge_evaluator.py b/sagemaker-train/tests/integ/train/test_llm_as_judge_evaluator.py index f7323694b5..10e453af0e 100644 --- a/sagemaker-train/tests/integ/train/test_llm_as_judge_evaluator.py +++ b/sagemaker-train/tests/integ/train/test_llm_as_judge_evaluator.py @@ -72,7 +72,7 @@ TEST_CONFIG = { "model_package_arn": "arn:aws:sagemaker:us-west-2:729646638167:model-package/sdk-test-finetuned-models/1", - "evaluator_model": "anthropic.claude-3-5-haiku-20241022-v1:0", + "evaluator_model": "anthropic.claude-sonnet-4-20250514-v1:0", "dataset_s3_uri": "s3://sagemaker-us-west-2-729646638167/model-customization/eval/gen_qa.jsonl", "builtin_metrics": ["Completeness", "Faithfulness"], "custom_metrics_json": json.dumps([CUSTOM_METRIC_DICT]), diff --git a/sagemaker-train/tests/unit/train/evaluate/test_llm_as_judge_evaluator.py b/sagemaker-train/tests/unit/train/evaluate/test_llm_as_judge_evaluator.py index edc05932b0..6a9cb05fd9 100644 --- a/sagemaker-train/tests/unit/train/evaluate/test_llm_as_judge_evaluator.py +++ b/sagemaker-train/tests/unit/train/evaluate/test_llm_as_judge_evaluator.py @@ -31,7 +31,7 @@ DEFAULT_MODEL_PACKAGE_GROUP_ARN = "arn:aws:sagemaker:us-west-2:123456789012:model-package-group/test-group" DEFAULT_BASE_MODEL_ARN = "arn:aws:sagemaker:us-west-2:aws:hub-content/SageMakerPublicHub/Model/llama3-2-1b-instruct/1.0.0" DEFAULT_ARTIFACT_ARN = "arn:aws:sagemaker:us-west-2:123456789012:artifact/test-artifact" -DEFAULT_EVALUATOR_MODEL = "anthropic.claude-3-5-sonnet-20240620-v1:0" +DEFAULT_EVALUATOR_MODEL = "anthropic.claude-sonnet-4-20250514-v1:0" @patch('sagemaker.train.common_utils.model_resolution._resolve_base_model') @@ -847,13 +847,21 @@ def test_llm_as_judge_evaluator_with_mlflow_names(mock_artifact, mock_resolve): @patch('sagemaker.core.resources.Artifact') def test_llm_as_judge_evaluator_valid_evaluator_models(mock_artifact, mock_resolve): """Test LLMAsJudgeEvaluator with valid evaluator models.""" + # Models available in us-west-2 per the regional allowlist. The Claude 3.5 + # Sonnet v1/v2 models are intentionally omitted here — they are only allowed + # in ap-northeast-1 (see test_llm_as_judge_evaluator_region_restriction). valid_models = [ - "anthropic.claude-3-5-sonnet-20240620-v1:0", - "anthropic.claude-3-5-sonnet-20241022-v2:0", "anthropic.claude-3-haiku-20240307-v1:0", - "anthropic.claude-3-5-haiku-20241022-v1:0", + "anthropic.claude-sonnet-4-20250514-v1:0", + "anthropic.claude-haiku-4-5-20251001-v1:0", + "anthropic.claude-sonnet-4-5-20250929-v1:0", + "anthropic.claude-opus-4-5-20251101-v1:0", "meta.llama3-1-70b-instruct-v1:0", "mistral.mistral-large-2402-v1:0", + "amazon.nova-pro-v1:0", + "amazon.nova-2-lite-v1:0", + "amazon.nova-micro-v1:0", + "amazon.nova-premier-v1:0", ] mock_info = Mock()