diff --git a/sagemaker-train/src/sagemaker/train/base_trainer.py b/sagemaker-train/src/sagemaker/train/base_trainer.py index ff3bdcd751..0d1795af1e 100644 --- a/sagemaker-train/src/sagemaker/train/base_trainer.py +++ b/sagemaker-train/src/sagemaker/train/base_trainer.py @@ -32,6 +32,7 @@ _validate_hyperparameter_values, _get_smhp_replicas_enum, ) +from sagemaker.train.common_utils.data_utils import validate_data_path_exists from sagemaker.train.common_utils.metrics_visualizer import plot_training_metrics from sagemaker.train.common_utils.mlflow_config_utils import resolve_mlflow_tracking_fields from sagemaker.train.common_utils.validator import validate_hyperpod_compute @@ -598,7 +599,7 @@ def _validate_instance_count(self, instance_count, sagemaker_session): return smhp_replicas_enum @abstractmethod - def train(self, input_data_config: List[InputData], wait: bool = True, logs: bool = True, wait_timeout: Optional[int] = None): + def train(self, input_data_config: List[InputData], wait: bool = True, logs: bool = True, wait_timeout: Optional[int] = None, dry_run: bool = False): """Common training method that calls the specific implementation.""" pass @@ -614,7 +615,7 @@ def _get_extra_smtj_hyperparameters(self) -> Dict[str, Any]: return {} def _train_serverful_smtj(self, training_dataset=None, validation_dataset=None, - wait=True, wait_timeout=None, poll=5): + wait=True, wait_timeout=None, poll=5, dry_run=False): """Execute training on serverful SageMaker Training Job (SMTJ) compute. Uses ModelTrainer.from_recipe() with the model's recipe template from @@ -967,6 +968,20 @@ def _yaml_safe_default(value): base_job_name=base_job_name, ) + # Validate data paths exist before submission + if resolved_training_dataset: + validate_data_path_exists( + resolved_training_dataset, sagemaker_session, label="training dataset" + ) + if resolved_validation_dataset: + validate_data_path_exists( + resolved_validation_dataset, sagemaker_session, label="validation dataset" + ) + + if dry_run: + logger.info("Dry-run validation passed. No job submitted.") + return None + # Execute training model_trainer.train( wait=wait, @@ -1085,7 +1100,7 @@ def _resolve_checkpoint_from_manifest( return checkpoint_path def _train_hyperpod(self, training_dataset=None, validation_dataset=None, - wait=True, wait_timeout=None, poll=5): + wait=True, wait_timeout=None, poll=5, dry_run=False): """Execute training on a SageMaker HyperPod cluster. Uses the HyperPod CLI to connect to the cluster and submit a training job @@ -1228,6 +1243,20 @@ def _train_hyperpod(self, training_dataset=None, validation_dataset=None, if getattr(self, 'model_source', None): override_parameters["recipes.run.model_name_or_path"] = self.model_source + # Validate data paths exist before submission + if resolved_training_dataset: + validate_data_path_exists( + resolved_training_dataset, sagemaker_session, label="training dataset" + ) + if resolved_validation_dataset: + validate_data_path_exists( + resolved_validation_dataset, sagemaker_session, label="validation dataset" + ) + + if dry_run: + logger.info("Dry-run validation passed. No job submitted.") + return None + # Submit job start_job_cmd = [ "hyperpod", "start-job", diff --git a/sagemaker-train/src/sagemaker/train/common_utils/data_utils.py b/sagemaker-train/src/sagemaker/train/common_utils/data_utils.py index b5db62c81b..9034df75d3 100644 --- a/sagemaker-train/src/sagemaker/train/common_utils/data_utils.py +++ b/sagemaker-train/src/sagemaker/train/common_utils/data_utils.py @@ -94,6 +94,146 @@ def load_file_content( raise FileLoadError(f"Failed to read file {file_path}: {e}") +def validate_data_path_exists( + data_path: str, + sagemaker_session, + label: str = "data", +) -> None: + """Validate that a data path (S3 URI or dataset ARN) exists and is accessible. + + Called inline during dry_run to catch bad paths before job submission. + + Args: + data_path: S3 URI or SageMaker hub-content DataSet ARN to validate. + sagemaker_session: SageMaker session (provides boto_session). + label: Human-readable label for error messages. + + Raises: + ValueError: If the path does not exist or is inaccessible. + """ + # Handle SageMaker hub-content DataSet ARNs + if data_path.startswith("arn:aws:sagemaker:") and "/DataSet/" in data_path: + pattern = ( + r"^arn:aws:sagemaker:([^:]+):(\d+):hub-content/" + r"([^/]+)/DataSet/([^/]+)/([\d\.]+)$" + ) + match = re.match(pattern, data_path) + if not match: + raise ValueError( + f"Invalid {label} DataSet ARN format: {data_path}" + ) + + _, _, hub_name, content_name, content_version = match.groups() + sm_client = sagemaker_session.sagemaker_client + + try: + sm_client.describe_hub_content( + HubName=hub_name, + HubContentType="DataSet", + HubContentName=content_name, + HubContentVersion=content_version, + ) + except ClientError as e: + code = e.response["Error"]["Code"] + if code == "ResourceNotFound" or "does not exist" in str(e).lower(): + raise ValueError( + f"{label.capitalize()} DataSet does not exist: {data_path}" + ) + elif "AccessDenied" in str(e): + logger.warning( + "Cannot verify %s DataSet %s from caller identity " + "(AccessDenied). The execution role may still have access.", + label, data_path, + ) + else: + raise ValueError( + f"Error validating {label} DataSet {data_path}: {e}" + ) + return + + # Handle S3 URIs + parts = _parse_s3_uri(data_path) + if parts is None: + raise ValueError( + f"Invalid {label} path format: {data_path}. " + f"Expected an S3 URI (s3://bucket/key) or a DataSet ARN." + ) + + bucket, key = parts + s3 = sagemaker_session.boto_session.client("s3") + + try: + resp = s3.list_objects_v2(Bucket=bucket, Prefix=key, MaxKeys=1) + if resp.get("KeyCount", 0) == 0: + raise ValueError( + f"S3 {label} path does not exist: {data_path}" + ) + except ClientError as e: + code = e.response["Error"]["Code"] + if code == "403" or "AccessDenied" in str(e): + # Caller may not have access but the execution role might — + # log a warning and allow the job to proceed. + logger.warning( + "Cannot verify S3 %s path %s from caller identity " + "(AccessDenied). The execution role may still have access.", + label, data_path, + ) + else: + raise ValueError(f"Error accessing S3 {label} path {data_path}: {e}") + + +def _validate_dataset_arn_exists( + dataset_arn: str, + sagemaker_session, + label: str = "data", +) -> None: + """Validate that a SageMaker hub-content DataSet ARN exists. + + Args: + dataset_arn: ARN like arn:aws:sagemaker:::hub-content//DataSet// + sagemaker_session: SageMaker session (provides boto_session). + label: Human-readable label for error messages. + + Raises: + ValueError: If the dataset ARN cannot be described. + """ + + pattern = ( + r"^arn:aws:sagemaker:([^:]+):(\d+):hub-content/" + r"([^/]+)/DataSet/([^/]+)/([\d\.]+)$" + ) + match = re.match(pattern, dataset_arn) + if not match: + raise ValueError( + f"Invalid {label} DataSet ARN format: {dataset_arn}" + ) + + region, _, hub_name, content_name, content_version = match.groups() + sm_client = sagemaker_session.sagemaker_client + + try: + sm_client.describe_hub_content( + HubName=hub_name, + HubContentType="DataSet", + HubContentName=content_name, + HubContentVersion=content_version, + ) + except ClientError as e: + code = e.response["Error"]["Code"] + if code == "ResourceNotFound" or "does not exist" in str(e).lower(): + raise ValueError( + f"{label.capitalize()} DataSet does not exist: {dataset_arn}" + ) + elif code == "AccessDeniedException" or "AccessDenied" in str(e): + raise ValueError( + f"Access denied for {label} DataSet: {dataset_arn}. " + "Check IAM permissions for sagemaker:DescribeHubContent." + ) + raise ValueError( + f"Error validating {label} DataSet {dataset_arn}: {e}" + ) + + def _has_multimodal_content(record: dict) -> bool: """Check if a single record contains multimodal content.""" if "messages" not in record: diff --git a/sagemaker-train/src/sagemaker/train/dpo_trainer.py b/sagemaker-train/src/sagemaker/train/dpo_trainer.py index 387e7b226a..beab096959 100644 --- a/sagemaker-train/src/sagemaker/train/dpo_trainer.py +++ b/sagemaker-train/src/sagemaker/train/dpo_trainer.py @@ -24,7 +24,7 @@ _validate_eula_for_gated_model, _validate_hyperparameter_values ) -from sagemaker.train.common_utils.data_utils import is_multimodal_data +from sagemaker.train.common_utils.data_utils import is_multimodal_data, validate_data_path_exists from sagemaker.core.telemetry.telemetry_logging import _telemetry_emitter from sagemaker.core.telemetry.constants import Feature from sagemaker.train.constants import get_sagemaker_hub_name @@ -217,7 +217,8 @@ def train(self, validation_dataset: Optional[Union[str, DataSet]] = None, wait: bool = True, wait_timeout: Optional[int] = None, - poll: int = 5): + poll: int = 5, + dry_run: bool = False): """Execute the DPO training job. Parameters: @@ -234,9 +235,13 @@ def train(self, If None, uses the default timeout from the wait utility. poll (int): Polling interval in seconds for checking training job status. Defaults to 5. + dry_run (bool): + If True, runs all validation (IAM, hyperparameters, infrastructure, data paths) + without submitting a job. Returns None on success, raises on validation failure. + Defaults to False. Returns: - TrainingJob: The SageMaker training job object. + TrainingJob: The SageMaker training job object, or None if dry_run=True. """ # Dispatch based on compute type if isinstance(self.compute, HyperPodCompute): @@ -246,6 +251,7 @@ def train(self, wait=wait, wait_timeout=wait_timeout, poll=poll, + dry_run=dry_run, ) elif isinstance(self.compute, TrainingJobCompute): return self._train_serverful_smtj( @@ -254,6 +260,7 @@ def train(self, wait=wait, wait_timeout=wait_timeout, poll=poll, + dry_run=dry_run, ) # Default: serverless compute (None) @@ -340,6 +347,22 @@ def train(self, if self.stopping_condition is not None: create_args["stopping_condition"] = self.stopping_condition + # Validate data paths exist before submission + effective_training = training_dataset or self.training_dataset + effective_validation = validation_dataset or self.validation_dataset + if effective_training and isinstance(effective_training, str): + validate_data_path_exists( + effective_training, sagemaker_session, label="training dataset" + ) + if effective_validation and isinstance(effective_validation, str): + validate_data_path_exists( + effective_validation, sagemaker_session, label="validation dataset" + ) + + if dry_run: + logger.info("Dry-run validation passed. No job submitted.") + return None + try: training_job = TrainingJob.create(**create_args) except Exception as e: diff --git a/sagemaker-train/src/sagemaker/train/evaluate/base_evaluator.py b/sagemaker-train/src/sagemaker/train/evaluate/base_evaluator.py index 29ab44133a..59a858e4fe 100644 --- a/sagemaker-train/src/sagemaker/train/evaluate/base_evaluator.py +++ b/sagemaker-train/src/sagemaker/train/evaluate/base_evaluator.py @@ -1034,15 +1034,22 @@ def get_resolved_recipe(self) -> Dict[str, Any]: object.__setattr__(self, '_resolved_recipe_cache', resolved) return copy.deepcopy(resolved) - def evaluate(self) -> Any: + def evaluate(self, dry_run: bool = False) -> Any: """Create and start an evaluation execution. This method must be implemented by subclasses to define the specific evaluation logic for different evaluation types (benchmark, custom scorer, LLM-as-judge, etc.). + Args: + dry_run (bool): + If True, runs all validation (IAM, model resolution, data paths) + without submitting the evaluation. Returns None on success, raises + on validation failure. Defaults to False. + Returns: - EvaluationPipelineExecution: The created evaluation execution object. + EvaluationPipelineExecution: The created evaluation execution object, + or None if dry_run=True. Raises: NotImplementedError: This is an abstract method that must be implemented by subclasses. diff --git a/sagemaker-train/src/sagemaker/train/evaluate/benchmark_evaluator.py b/sagemaker-train/src/sagemaker/train/evaluate/benchmark_evaluator.py index a9c49a5b9d..d3eaf8f787 100644 --- a/sagemaker-train/src/sagemaker/train/evaluate/benchmark_evaluator.py +++ b/sagemaker-train/src/sagemaker/train/evaluate/benchmark_evaluator.py @@ -591,7 +591,7 @@ def _get_benchmark_template_additions(self, eval_subtask: Optional[Union[str, Li return benchmark_context @_telemetry_emitter(feature=Feature.MODEL_CUSTOMIZATION, func_name="BenchMarkEvaluator.evaluate") - def evaluate(self, subtask: Optional[Union[str, List[str]]] = None) -> EvaluationPipelineExecution: + def evaluate(self, subtask: Optional[Union[str, List[str]]] = None, dry_run: bool = False) -> EvaluationPipelineExecution: """Create and start a benchmark evaluation job. Supports multiple compute backends via the ``compute`` parameter set at @@ -601,12 +601,17 @@ def evaluate(self, subtask: Optional[Union[str, List[str]]] = None) -> Evaluatio - **HyperPod**: Submits to a HyperPod cluster via the HyperPod CLI. Args: - subtask (Optional[Union[str, list[str]]]): Optional subtask(s) to evaluate. If not provided, - uses the subtasks from constructor. Can be a single subtask string, a list of - subtasks, or 'ALL' to run all subtasks. + subtask (Optional[Union[str, list[str]]]): Optional subtask(s) to evaluate. + If not provided, uses the subtasks from constructor. Can be a single + subtask string, a list of subtasks, or 'ALL' to run all subtasks. + dry_run (bool): + If True, runs all validation (IAM, model resolution, data paths) + without submitting the evaluation. Returns None on success, raises + on validation failure. Defaults to False. Returns: - EvaluationPipelineExecution: The created benchmark evaluation execution. + EvaluationPipelineExecution: The created benchmark evaluation execution, + or None if dry_run=True. Example: @@ -702,7 +707,11 @@ def evaluate(self, subtask: Optional[Union[str, List[str]]] = None) -> Evaluatio # Generate execution name name = self.base_eval_name or f"benchmark-eval-{self.benchmark.value}" - + + if dry_run: + _logger.info("Dry-run validation passed. No evaluation submitted.") + return None + # Start execution return self._start_execution( eval_type=EvalType.BENCHMARK, diff --git a/sagemaker-train/src/sagemaker/train/evaluate/custom_scorer_evaluator.py b/sagemaker-train/src/sagemaker/train/evaluate/custom_scorer_evaluator.py index 022e33b351..14fbd9e278 100644 --- a/sagemaker-train/src/sagemaker/train/evaluate/custom_scorer_evaluator.py +++ b/sagemaker-train/src/sagemaker/train/evaluate/custom_scorer_evaluator.py @@ -16,7 +16,9 @@ from .execution import EvaluationPipelineExecution from sagemaker.core.telemetry.telemetry_logging import _telemetry_emitter from sagemaker.core.telemetry.constants import Feature +from sagemaker.train.common_utils.data_utils import validate_data_path_exists from sagemaker.train.constants import get_sagemaker_hub_name +from sagemaker.train.defaults import TrainDefaults _logger = logging.getLogger(__name__) @@ -390,7 +392,7 @@ def _get_inference_params_from_hub(self, region: str) -> dict: return fallback_params @_telemetry_emitter(feature=Feature.MODEL_CUSTOMIZATION, func_name="CustomScorerEvaluator.evaluate") - def evaluate(self) -> EvaluationPipelineExecution: + def evaluate(self, dry_run: bool = False) -> EvaluationPipelineExecution: """Create and start a custom scorer evaluation job. Supports multiple compute backends via the ``compute`` parameter set at @@ -398,9 +400,16 @@ def evaluate(self) -> EvaluationPipelineExecution: - **Serverless** (default): Runs via SageMaker Pipelines. - **SMTJ**: Runs on user-managed instances via ModelTrainer. - **HyperPod**: Submits to a HyperPod cluster via the HyperPod CLI. - + + Args: + dry_run (bool): + If True, runs all validation (IAM, model resolution, data paths) + without submitting the evaluation. Returns None on success, raises + on validation failure. Defaults to False. + Returns: - EvaluationPipelineExecution: The created custom scorer evaluation execution + EvaluationPipelineExecution: The created custom scorer evaluation execution, + or None if dry_run=True. Example: .. code:: python @@ -487,7 +496,20 @@ def evaluate(self) -> EvaluationPipelineExecution: # Generate execution name name = self.base_eval_name or f"custom-scorer-eval" - + + # Validate dataset path exists + if hasattr(self, 'dataset') and self.dataset and isinstance(self.dataset, str): + session = TrainDefaults.get_sagemaker_session( + sagemaker_session=self.sagemaker_session + ) + validate_data_path_exists( + self.dataset, session, label="evaluation dataset" + ) + + if dry_run: + _logger.info("Dry-run validation passed. No evaluation submitted.") + return None + # Start execution return self._start_execution( eval_type=EvalType.CUSTOM_SCORER, diff --git a/sagemaker-train/src/sagemaker/train/evaluate/llm_as_judge_evaluator.py b/sagemaker-train/src/sagemaker/train/evaluate/llm_as_judge_evaluator.py index ccf852468a..27bbc0ae82 100644 --- a/sagemaker-train/src/sagemaker/train/evaluate/llm_as_judge_evaluator.py +++ b/sagemaker-train/src/sagemaker/train/evaluate/llm_as_judge_evaluator.py @@ -20,9 +20,11 @@ ) from sagemaker.core.telemetry.telemetry_logging import _telemetry_emitter from sagemaker.core.telemetry.constants import Feature +from sagemaker.train.common_utils.data_utils import validate_data_path_exists from sagemaker.train.common_utils.model_aliases import NOVA_BEDROCK_MODEL_IDS from sagemaker.train.common_utils.recipe_utils import _is_nova_model from sagemaker.train.constants import _ALLOWED_EVALUATOR_MODELS +from sagemaker.train.defaults import TrainDefaults _logger = logging.getLogger(__name__) @@ -751,7 +753,7 @@ def _get_llmaj_template_additions(self, eval_name: str) -> dict: } @_telemetry_emitter(feature=Feature.MODEL_CUSTOMIZATION, func_name="LLMAsJudgeEvaluator.evaluate") - def evaluate(self): + def evaluate(self, dry_run: bool = False): """Create and start an LLM-as-judge evaluation job. This method initiates a 2-phase evaluation job: @@ -764,9 +766,16 @@ def evaluate(self): inference responses and writes them to S3. Phase 2 remains unchanged — the LLMAJEvaluation judging step evaluates those responses with the judge model. - + + Args: + dry_run (bool): + If True, runs all validation (IAM, model resolution, data paths) + without submitting the evaluation. Returns None on success, raises + on validation failure. Defaults to False. + Returns: - EvaluationPipelineExecution: The created LLM-as-judge evaluation execution + EvaluationPipelineExecution: The created LLM-as-judge evaluation execution, + or None if dry_run=True. Raises: ValueError: If invalid model, dataset, or metric configurations are provided @@ -942,7 +951,20 @@ def evaluate(self): # Render pipeline definition pipeline_definition = self._render_pipeline_definition(template_str, template_context) - + + # Validate dataset path exists + if hasattr(self, 'dataset') and self.dataset and isinstance(self.dataset, str): + session = TrainDefaults.get_sagemaker_session( + sagemaker_session=self.sagemaker_session + ) + validate_data_path_exists( + self.dataset, session, label="evaluation dataset" + ) + + if dry_run: + _logger.info("Dry-run validation passed. No evaluation submitted.") + return None + # Start execution (name already generated earlier) return self._start_execution( eval_type=EvalType.LLM_AS_JUDGE, diff --git a/sagemaker-train/src/sagemaker/train/rlaif_trainer.py b/sagemaker-train/src/sagemaker/train/rlaif_trainer.py index 50722f6d80..7929b78493 100644 --- a/sagemaker-train/src/sagemaker/train/rlaif_trainer.py +++ b/sagemaker-train/src/sagemaker/train/rlaif_trainer.py @@ -25,7 +25,7 @@ _validate_eula_for_gated_model, _validate_hyperparameter_values ) -from sagemaker.train.common_utils.data_utils import is_multimodal_data +from sagemaker.train.common_utils.data_utils import is_multimodal_data, validate_data_path_exists from sagemaker.core.telemetry.telemetry_logging import _telemetry_emitter from sagemaker.core.telemetry.constants import Feature from sagemaker.train.constants import get_sagemaker_hub_name, _ALLOWED_REWARD_MODEL_IDS @@ -203,7 +203,7 @@ def _validate_reward_model_id(self, reward_model_id): @_telemetry_emitter(feature=Feature.MODEL_CUSTOMIZATION, func_name="RLAIFTrainer.train") - def train(self, training_dataset: Optional[Union[str, DataSet]] = None, validation_dataset: Optional[Union[str, DataSet]] = None, wait: bool = True, wait_timeout: Optional[int] = None, poll: int = 5): + def train(self, training_dataset: Optional[Union[str, DataSet]] = None, validation_dataset: Optional[Union[str, DataSet]] = None, wait: bool = True, wait_timeout: Optional[int] = None, poll: int = 5, dry_run: bool = False): """Execute the RLAIF training job. Parameters: @@ -220,9 +220,13 @@ def train(self, training_dataset: Optional[Union[str, DataSet]] = None, validati If None, uses the default timeout from the wait utility. poll (int): Polling interval in seconds for checking training job status. Defaults to 5. + dry_run (bool): + If True, runs all validation (IAM, hyperparameters, infrastructure, data paths) + without submitting a job. Returns None on success, raises on validation failure. + Defaults to False. Returns: - TrainingJob: The SageMaker training job object. + TrainingJob: The SageMaker training job object, or None if dry_run=True. """ sagemaker_session = TrainDefaults.get_sagemaker_session( sagemaker_session=self.sagemaker_session @@ -302,6 +306,22 @@ def train(self, training_dataset: Optional[Union[str, DataSet]] = None, validati if self.stopping_condition is not None: create_args["stopping_condition"] = self.stopping_condition + # Validate data paths exist before submission + effective_training = training_dataset or self.training_dataset + effective_validation = validation_dataset or self.validation_dataset + if effective_training and isinstance(effective_training, str): + validate_data_path_exists( + effective_training, sagemaker_session, label="training dataset" + ) + if effective_validation and isinstance(effective_validation, str): + validate_data_path_exists( + effective_validation, sagemaker_session, label="validation dataset" + ) + + if dry_run: + logger.info("Dry-run validation passed. No job submitted.") + return None + try: training_job = TrainingJob.create(**create_args) except Exception as e: diff --git a/sagemaker-train/src/sagemaker/train/rlvr_trainer.py b/sagemaker-train/src/sagemaker/train/rlvr_trainer.py index 0c0670425f..860529a825 100644 --- a/sagemaker-train/src/sagemaker/train/rlvr_trainer.py +++ b/sagemaker-train/src/sagemaker/train/rlvr_trainer.py @@ -30,7 +30,7 @@ _validate_eula_for_gated_model, _validate_hyperparameter_values ) -from sagemaker.train.common_utils.data_utils import is_multimodal_data, load_file_content +from sagemaker.train.common_utils.data_utils import is_multimodal_data, load_file_content, validate_data_path_exists from sagemaker.train.common_utils.rlvr_reward_verifier import verify_reward_function from sagemaker.core.telemetry.telemetry_logging import _telemetry_emitter from sagemaker.core.telemetry.constants import Feature @@ -365,7 +365,7 @@ def _verify_reward_function( @_telemetry_emitter(feature=Feature.MODEL_CUSTOMIZATION, func_name="RLVRTrainer.train") def train(self, training_dataset: Optional[Union[str, DataSet]] = None, - validation_dataset: Optional[Union[str, DataSet]] = None, wait: bool = True, wait_timeout: Optional[int] = None, poll: int = 5): + validation_dataset: Optional[Union[str, DataSet]] = None, wait: bool = True, wait_timeout: Optional[int] = None, poll: int = 5, dry_run: bool = False): """Execute the RLVR training job. Parameters: @@ -382,9 +382,13 @@ def train(self, training_dataset: Optional[Union[str, DataSet]] = None, If None, uses the default timeout from the wait utility. poll (int): Polling interval in seconds for checking training job status. Defaults to 5. + dry_run (bool): + If True, runs all validation (IAM, hyperparameters, infrastructure, data paths) + without submitting a job. Returns None on success, raises on validation failure. + Defaults to False. Returns: - TrainingJob: The SageMaker training job object. + TrainingJob: The SageMaker training job object, or None if dry_run=True. """ # Dispatch based on compute type if isinstance(self.compute, HyperPodCompute): @@ -394,6 +398,7 @@ def train(self, training_dataset: Optional[Union[str, DataSet]] = None, wait=wait, wait_timeout=wait_timeout, poll=poll, + dry_run=dry_run, ) elif isinstance(self.compute, TrainingJobCompute): return self._train_serverful_smtj( @@ -402,6 +407,7 @@ def train(self, training_dataset: Optional[Union[str, DataSet]] = None, wait=wait, wait_timeout=wait_timeout, poll=poll, + dry_run=dry_run, ) # Default: serverless compute (None) @@ -494,6 +500,22 @@ def train(self, training_dataset: Optional[Union[str, DataSet]] = None, if self.stopping_condition is not None: create_args["stopping_condition"] = self.stopping_condition + # Validate data paths exist before submission + effective_training = training_dataset or self.training_dataset + effective_validation = validation_dataset or self.validation_dataset + if effective_training and isinstance(effective_training, str): + validate_data_path_exists( + effective_training, sagemaker_session, label="training dataset" + ) + if effective_validation and isinstance(effective_validation, str): + validate_data_path_exists( + effective_validation, sagemaker_session, label="validation dataset" + ) + + if dry_run: + logger.info("Dry-run validation passed. No job submitted.") + return None + try: training_job = TrainingJob.create(**create_args) except Exception as e: diff --git a/sagemaker-train/src/sagemaker/train/sft_trainer.py b/sagemaker-train/src/sagemaker/train/sft_trainer.py index f16e097d8e..0d5298e9eb 100644 --- a/sagemaker-train/src/sagemaker/train/sft_trainer.py +++ b/sagemaker-train/src/sagemaker/train/sft_trainer.py @@ -25,7 +25,7 @@ _validate_eula_for_gated_model, _validate_hyperparameter_values ) -from sagemaker.train.common_utils.data_utils import is_multimodal_data +from sagemaker.train.common_utils.data_utils import is_multimodal_data, validate_data_path_exists from sagemaker.train.common_utils.data_mixing_utils import ( validate_data_mixing_model, validate_data_mixing_categories, @@ -243,7 +243,7 @@ def _process_hyperparameters(self): self.hyperparameters._specs.pop('validation_data_path', None) @_telemetry_emitter(feature=Feature.MODEL_CUSTOMIZATION, func_name="SFTTrainer.train") - def train(self, training_dataset: Optional[Union[str, DataSet]] = None, validation_dataset: Optional[Union[str, DataSet]] = None, wait: bool = True, wait_timeout: Optional[int] = None, poll: int = 5): + def train(self, training_dataset: Optional[Union[str, DataSet]] = None, validation_dataset: Optional[Union[str, DataSet]] = None, wait: bool = True, wait_timeout: Optional[int] = None, poll: int = 5, dry_run: bool = False): """Execute the SFT training job. Parameters: @@ -260,9 +260,13 @@ def train(self, training_dataset: Optional[Union[str, DataSet]] = None, validati If None, uses the default timeout from the wait utility. poll (int): Polling interval in seconds for checking training job status. Defaults to 5. + dry_run (bool): + If True, runs all validation (IAM, hyperparameters, infrastructure, data paths) + without submitting a job. Returns None on success, raises on validation failure. + Defaults to False. Returns: - TrainingJob: The SageMaker training job object. + TrainingJob: The SageMaker training job object, or None if dry_run=True. """ # Dispatch based on compute type if isinstance(self.compute, HyperPodCompute): @@ -299,6 +303,7 @@ def train(self, training_dataset: Optional[Union[str, DataSet]] = None, validati wait=wait, wait_timeout=wait_timeout, poll=poll, + dry_run=dry_run, ) elif isinstance(self.compute, TrainingJobCompute): if self.data_mixing_config is not None: @@ -309,6 +314,7 @@ def train(self, training_dataset: Optional[Union[str, DataSet]] = None, validati wait=wait, wait_timeout=wait_timeout, poll=poll, + dry_run=dry_run, ) # Default: serverless compute (None) @@ -407,6 +413,22 @@ def train(self, training_dataset: Optional[Union[str, DataSet]] = None, validati if self.stopping_condition is not None: create_args["stopping_condition"] = self.stopping_condition + # Validate data paths exist before submission + effective_training = training_dataset or self.training_dataset + effective_validation = validation_dataset or self.validation_dataset + if effective_training and isinstance(effective_training, str): + validate_data_path_exists( + effective_training, sagemaker_session, label="training dataset" + ) + if effective_validation and isinstance(effective_validation, str): + validate_data_path_exists( + effective_validation, sagemaker_session, label="validation dataset" + ) + + if dry_run: + logger.info("Dry-run validation passed. No job submitted.") + return None + try: training_job = TrainingJob.create(**create_args) except Exception as e: diff --git a/sagemaker-train/tests/integ/train/test_dry_run_integration.py b/sagemaker-train/tests/integ/train/test_dry_run_integration.py new file mode 100644 index 0000000000..d42049421b --- /dev/null +++ b/sagemaker-train/tests/integ/train/test_dry_run_integration.py @@ -0,0 +1,188 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Integration tests for dry_run=True on trainers. + +These tests validate that dry_run performs real validation against AWS +(IAM role resolution, S3 path existence, hyperparameter constraints) +without consuming compute. No training jobs are submitted. + +A small sample dataset is uploaded to the SageMaker default bucket +during test setup and cleaned up afterward. +""" +from __future__ import absolute_import + +import json +import time +import random + +import pytest + +from sagemaker.train.sft_trainer import SFTTrainer +from sagemaker.train.dpo_trainer import DPOTrainer +from sagemaker.train.rlvr_trainer import RLVRTrainer +from sagemaker.train.common import TrainingType +from sagemaker.train.evaluate.benchmark_evaluator import BenchMarkEvaluator + + +MODEL_PACKAGE_GROUP = ( + "arn:aws:sagemaker:us-west-2:729646638167:" + "model-package-group/sdk-test-finetuned-models" +) +MODEL_ID = "meta-textgeneration-llama-3-2-1b-instruct" +DATASET_KEY = "dry-run-integ-test/sample_train.jsonl" + + +@pytest.fixture(scope="module") +def valid_dataset(sagemaker_session): + """Upload a small sample dataset to the default bucket, return its S3 URI.""" + bucket = sagemaker_session.default_bucket() + s3 = sagemaker_session.boto_session.client("s3") + + samples = [ + {"messages": [ + {"role": "user", "content": [{"text": "What is 2+2?"}]}, + {"role": "assistant", "content": [{"text": "4"}]}, + ]}, + {"messages": [ + {"role": "user", "content": [{"text": "Capital of France?"}]}, + {"role": "assistant", "content": [{"text": "Paris"}]}, + ]}, + ] + body = "\n".join(json.dumps(s) for s in samples) + s3.put_object(Bucket=bucket, Key=DATASET_KEY, Body=body.encode("utf-8")) + + s3_uri = f"s3://{bucket}/{DATASET_KEY}" + yield s3_uri + + # Cleanup + s3.delete_object(Bucket=bucket, Key=DATASET_KEY) + + +@pytest.fixture(scope="module") +def nonexistent_dataset(sagemaker_session): + """Return an S3 URI in the default bucket that does not exist.""" + bucket = sagemaker_session.default_bucket() + return f"s3://{bucket}/dry-run-integ-test/nonexistent_path_12345.jsonl" + + +class TestDryRunS3PathValidation: + """Verify dry_run raises when S3 data paths do not exist.""" + + def test_sft_fails_on_nonexistent_training_dataset( + self, sagemaker_session, nonexistent_dataset + ): + trainer = SFTTrainer( + model=MODEL_ID, + training_type=TrainingType.LORA, + model_package_group=MODEL_PACKAGE_GROUP, + training_dataset=nonexistent_dataset, + accept_eula=True, + ) + + with pytest.raises(ValueError, match="does not exist"): + trainer.train(dry_run=True) + + def test_sft_fails_on_nonexistent_validation_dataset( + self, sagemaker_session, valid_dataset, nonexistent_dataset + ): + trainer = SFTTrainer( + model=MODEL_ID, + training_type=TrainingType.LORA, + model_package_group=MODEL_PACKAGE_GROUP, + training_dataset=valid_dataset, + validation_dataset=nonexistent_dataset, + accept_eula=True, + ) + + with pytest.raises(ValueError, match="does not exist"): + trainer.train(dry_run=True) + + +class TestDryRunPassesWithValidInputs: + """Verify dry_run=True returns None and does not create a job.""" + + def test_sft_dry_run_returns_none(self, sagemaker_session, valid_dataset): + trainer = SFTTrainer( + model=MODEL_ID, + training_type=TrainingType.LORA, + model_package_group=MODEL_PACKAGE_GROUP, + training_dataset=valid_dataset, + accept_eula=True, + ) + + result = trainer.train(dry_run=True) + assert result is None + + def test_dpo_dry_run_returns_none(self, sagemaker_session, valid_dataset): + trainer = DPOTrainer( + model=MODEL_ID, + training_type=TrainingType.LORA, + model_package_group=MODEL_PACKAGE_GROUP, + training_dataset=valid_dataset, + accept_eula=True, + ) + + result = trainer.train(dry_run=True) + assert result is None + + def test_rlvr_dry_run_returns_none(self, sagemaker_session, valid_dataset): + trainer = RLVRTrainer( + model=MODEL_ID, + training_type=TrainingType.LORA, + model_package_group=MODEL_PACKAGE_GROUP, + training_dataset=valid_dataset, + accept_eula=True, + ) + + result = trainer.train(dry_run=True) + assert result is None + + def test_no_training_job_created(self, sagemaker_session, valid_dataset): + """Confirm via the SageMaker API that no job was submitted.""" + unique_id = f"{int(time.time())}-{random.randint(10000, 99999)}" + base_name = f"dry-run-noop-{unique_id}" + + trainer = SFTTrainer( + model=MODEL_ID, + training_type=TrainingType.LORA, + model_package_group=MODEL_PACKAGE_GROUP, + training_dataset=valid_dataset, + accept_eula=True, + base_job_name=base_name, + ) + + trainer.train(dry_run=True) + + sm_client = sagemaker_session.sagemaker_client + response = sm_client.list_training_jobs( + NameContains=base_name, + MaxResults=1, + ) + assert len(response.get("TrainingJobSummaries", [])) == 0 + + +class TestEvaluateDryRun: + """Verify evaluate(dry_run=True) validates without submitting.""" + + def test_benchmark_evaluate_dry_run_returns_none(self, sagemaker_session): + from sagemaker.train.evaluate import get_benchmarks + Benchmark = get_benchmarks() + + evaluator = BenchMarkEvaluator( + benchmark=Benchmark.MMLU, + model=MODEL_ID, + s3_output_path=f"s3://{sagemaker_session.default_bucket()}/dry-run-eval-output/", + ) + + result = evaluator.evaluate(dry_run=True) + assert result is None diff --git a/sagemaker-train/tests/unit/train/common_utils/test_data_utils.py b/sagemaker-train/tests/unit/train/common_utils/test_data_utils.py index d0f9ae777b..8d6b8f87c3 100644 --- a/sagemaker-train/tests/unit/train/common_utils/test_data_utils.py +++ b/sagemaker-train/tests/unit/train/common_utils/test_data_utils.py @@ -230,5 +230,72 @@ def test_load_file_content_called_with_correct_args_json(self, mock_load): ) +class TestValidateDataPathExists(unittest.TestCase): + """Tests for validate_data_path_exists utility.""" + + def _make_session(self): + from unittest.mock import Mock + session = Mock() + self.s3 = Mock() + session.boto_session.client.return_value = self.s3 + from botocore.exceptions import ClientError + self.s3.exceptions.ClientError = ClientError + return session + + def test_object_exists(self): + from sagemaker.train.common_utils.data_utils import validate_data_path_exists + + session = self._make_session() + self.s3.list_objects_v2.return_value = {"KeyCount": 1} + + validate_data_path_exists("s3://my-bucket/data/train.jsonl", session, label="training") + self.s3.list_objects_v2.assert_called_once_with( + Bucket="my-bucket", Prefix="data/train.jsonl", MaxKeys=1 + ) + + def test_prefix_exists(self): + from sagemaker.train.common_utils.data_utils import validate_data_path_exists + + session = self._make_session() + self.s3.list_objects_v2.return_value = {"KeyCount": 3} + + validate_data_path_exists("s3://my-bucket/data/prefix/", session, label="training") + + def test_path_does_not_exist_raises(self): + from sagemaker.train.common_utils.data_utils import validate_data_path_exists + + session = self._make_session() + self.s3.list_objects_v2.return_value = {"KeyCount": 0} + + with self.assertRaises(ValueError) as ctx: + validate_data_path_exists( + "s3://my-bucket/bad/path.jsonl", session, label="training dataset" + ) + self.assertIn("does not exist", str(ctx.exception)) + + def test_access_denied_warns_not_raises(self): + from sagemaker.train.common_utils.data_utils import validate_data_path_exists + from botocore.exceptions import ClientError + + session = self._make_session() + self.s3.list_objects_v2.side_effect = ClientError( + {"Error": {"Code": "403", "Message": "Forbidden"}}, "ListObjectsV2" + ) + + # Should not raise — caller may lack access but execution role might have it + validate_data_path_exists("s3://locked-bucket/data.jsonl", session, label="data") + + def test_unrecognized_format_raises(self): + from sagemaker.train.common_utils.data_utils import validate_data_path_exists + from unittest.mock import Mock + + session = Mock() + with self.assertRaises(ValueError) as ctx: + validate_data_path_exists( + "arn:aws:sagemaker:us-east-1:123:dataset/foo", session + ) + self.assertIn("Invalid", str(ctx.exception)) + + if __name__ == "__main__": unittest.main() diff --git a/sagemaker-train/tests/unit/train/test_dpo_trainer.py b/sagemaker-train/tests/unit/train/test_dpo_trainer.py index e960451196..f7e65eae7c 100644 --- a/sagemaker-train/tests/unit/train/test_dpo_trainer.py +++ b/sagemaker-train/tests/unit/train/test_dpo_trainer.py @@ -621,3 +621,54 @@ def test_s3_model_without_base_model_name_raises(self, mock_resolve, mock_valida compute=HyperPodCompute(cluster_name="my-cluster", node_count=4), training_dataset="s3://bucket/train.jsonl", ) + + +class TestDPOTrainerDryRun: + """Tests for DPOTrainer.train(dry_run=True).""" + + @patch('sagemaker.train.dpo_trainer._validate_and_resolve_model_package_group') + @patch('sagemaker.train.dpo_trainer._get_fine_tuning_options_and_model_arn') + @patch('sagemaker.train.dpo_trainer.TrainDefaults.get_role') + @patch('sagemaker.train.dpo_trainer.TrainDefaults.get_sagemaker_session') + @patch('sagemaker.train.dpo_trainer._get_unique_name') + @patch('sagemaker.train.dpo_trainer._create_input_data_config') + @patch('sagemaker.train.dpo_trainer._convert_input_data_to_channels') + @patch('sagemaker.train.dpo_trainer._create_output_config') + @patch('sagemaker.train.dpo_trainer._create_serverless_config') + @patch('sagemaker.train.dpo_trainer._create_mlflow_config') + @patch('sagemaker.train.dpo_trainer._create_model_package_config') + @patch('sagemaker.train.dpo_trainer._validate_hyperparameter_values') + @patch('sagemaker.core.resources.TrainingJob.create') + @patch('sagemaker.train.common_utils.data_utils.validate_data_path_exists') + def test_dry_run_returns_none_without_submitting( + self, mock_validate_s3, mock_create, mock_validate_hp, mock_model_pkg, + mock_mlflow, mock_serverless, mock_output, mock_channels, mock_input, + mock_name, mock_session, mock_role, mock_options, mock_group, + ): + mock_group.return_value = "test-group" + mock_hp = Mock() + mock_hp.to_dict.return_value = {} + mock_hp._specs = {} + mock_options.return_value = (mock_hp, "model-arn", False) + + sess = Mock() + sess.boto_session.region_name = "us-east-1" + mock_session.return_value = sess + mock_role.return_value = "test-role" + mock_name.return_value = "job-name" + mock_input.return_value = [Mock()] + mock_channels.return_value = [Mock()] + mock_output.return_value = Mock() + mock_serverless.return_value = Mock() + mock_mlflow.return_value = Mock() + mock_model_pkg.return_value = Mock() + + trainer = DPOTrainer( + model="test-model", model_package_group="test-group", + training_dataset="s3://bucket/train.jsonl", + ) + trainer.train(dry_run=True) + + mock_create.assert_not_called() + mock_role.assert_called_once() + mock_validate_hp.assert_called_once() diff --git a/sagemaker-train/tests/unit/train/test_rlaif_trainer.py b/sagemaker-train/tests/unit/train/test_rlaif_trainer.py index a4f16fee42..f6cb5c4bdd 100644 --- a/sagemaker-train/tests/unit/train/test_rlaif_trainer.py +++ b/sagemaker-train/tests/unit/train/test_rlaif_trainer.py @@ -682,4 +682,55 @@ def test_train_wait_false_skips_wait(self, mock_training_job_create, mock_model_ trainer = RLAIFTrainer(model="test-model", model_package_group="test-group", training_dataset="s3://bucket/train") trainer.train(wait=False, wait_timeout=600) - mock_wait.assert_not_called() \ No newline at end of file + mock_wait.assert_not_called() + + +class TestRLAIFTrainerDryRun: + """Tests for RLAIFTrainer.train(dry_run=True).""" + + @patch('sagemaker.train.rlaif_trainer._validate_and_resolve_model_package_group') + @patch('sagemaker.train.rlaif_trainer._get_fine_tuning_options_and_model_arn') + @patch('sagemaker.train.rlaif_trainer.TrainDefaults.get_role') + @patch('sagemaker.train.rlaif_trainer.TrainDefaults.get_sagemaker_session') + @patch('sagemaker.train.rlaif_trainer._get_unique_name') + @patch('sagemaker.train.rlaif_trainer._create_input_data_config') + @patch('sagemaker.train.rlaif_trainer._convert_input_data_to_channels') + @patch('sagemaker.train.rlaif_trainer._create_output_config') + @patch('sagemaker.train.rlaif_trainer._create_serverless_config') + @patch('sagemaker.train.rlaif_trainer._create_mlflow_config') + @patch('sagemaker.train.rlaif_trainer._create_model_package_config') + @patch('sagemaker.train.rlaif_trainer._validate_hyperparameter_values') + @patch('sagemaker.core.resources.TrainingJob.create') + @patch('sagemaker.train.common_utils.data_utils.validate_data_path_exists') + def test_dry_run_returns_none_without_submitting( + self, mock_validate_s3, mock_create, mock_validate_hp, mock_model_pkg, + mock_mlflow, mock_serverless, mock_output, mock_channels, mock_input, + mock_name, mock_session, mock_role, mock_options, mock_group, + ): + mock_group.return_value = "test-group" + mock_hp = Mock() + mock_hp.to_dict.return_value = {} + mock_hp._specs = {} + mock_options.return_value = (mock_hp, "model-arn", False) + + sess = Mock() + sess.boto_session.region_name = "us-east-1" + mock_session.return_value = sess + mock_role.return_value = "test-role" + mock_name.return_value = "job-name" + mock_input.return_value = [Mock()] + mock_channels.return_value = [Mock()] + mock_output.return_value = Mock() + mock_serverless.return_value = Mock() + mock_mlflow.return_value = Mock() + mock_model_pkg.return_value = Mock() + + trainer = RLAIFTrainer( + model="test-model", model_package_group="test-group", + training_dataset="s3://bucket/train.jsonl", + ) + trainer.train(dry_run=True) + + mock_create.assert_not_called() + mock_role.assert_called_once() + mock_validate_hp.assert_called_once() diff --git a/sagemaker-train/tests/unit/train/test_rlvr_trainer.py b/sagemaker-train/tests/unit/train/test_rlvr_trainer.py index 5a6178392b..3f56ec78fd 100644 --- a/sagemaker-train/tests/unit/train/test_rlvr_trainer.py +++ b/sagemaker-train/tests/unit/train/test_rlvr_trainer.py @@ -620,3 +620,54 @@ def test_s3_model_without_base_model_name_raises(self, mock_resolve, mock_valida compute=HyperPodCompute(cluster_name="my-cluster", node_count=4), training_dataset="s3://bucket/train.jsonl", ) + + +class TestRLVRTrainerDryRun: + """Tests for RLVRTrainer.train(dry_run=True).""" + + @patch('sagemaker.train.rlvr_trainer._validate_and_resolve_model_package_group') + @patch('sagemaker.train.rlvr_trainer._get_fine_tuning_options_and_model_arn') + @patch('sagemaker.train.rlvr_trainer.TrainDefaults.get_role') + @patch('sagemaker.train.rlvr_trainer.TrainDefaults.get_sagemaker_session') + @patch('sagemaker.train.rlvr_trainer._get_unique_name') + @patch('sagemaker.train.rlvr_trainer._create_input_data_config') + @patch('sagemaker.train.rlvr_trainer._convert_input_data_to_channels') + @patch('sagemaker.train.rlvr_trainer._create_output_config') + @patch('sagemaker.train.rlvr_trainer._create_serverless_config') + @patch('sagemaker.train.rlvr_trainer._create_mlflow_config') + @patch('sagemaker.train.rlvr_trainer._create_model_package_config') + @patch('sagemaker.train.rlvr_trainer._validate_hyperparameter_values') + @patch('sagemaker.core.resources.TrainingJob.create') + @patch('sagemaker.train.common_utils.data_utils.validate_data_path_exists') + def test_dry_run_returns_none_without_submitting( + self, mock_validate_s3, mock_create, mock_validate_hp, mock_model_pkg, + mock_mlflow, mock_serverless, mock_output, mock_channels, mock_input, + mock_name, mock_session, mock_role, mock_options, mock_group, + ): + mock_group.return_value = "test-group" + mock_hp = Mock() + mock_hp.to_dict.return_value = {} + mock_hp._specs = {} + mock_options.return_value = (mock_hp, "model-arn", False) + + sess = Mock() + sess.boto_session.region_name = "us-east-1" + mock_session.return_value = sess + mock_role.return_value = "test-role" + mock_name.return_value = "job-name" + mock_input.return_value = [Mock()] + mock_channels.return_value = [Mock()] + mock_output.return_value = Mock() + mock_serverless.return_value = Mock() + mock_mlflow.return_value = Mock() + mock_model_pkg.return_value = Mock() + + trainer = RLVRTrainer( + model="test-model", model_package_group="test-group", + training_dataset="s3://bucket/train.jsonl", + ) + trainer.train(dry_run=True) + + mock_create.assert_not_called() + mock_role.assert_called_once() + mock_validate_hp.assert_called_once() diff --git a/sagemaker-train/tests/unit/train/test_sft_trainer.py b/sagemaker-train/tests/unit/train/test_sft_trainer.py index 7586a8d7df..a227a815dc 100644 --- a/sagemaker-train/tests/unit/train/test_sft_trainer.py +++ b/sagemaker-train/tests/unit/train/test_sft_trainer.py @@ -1284,3 +1284,81 @@ def test_disable_output_compression_stored(self, mock_resolve, mock_validate_gro ) assert trainer.disable_output_compression is True + + +class TestSFTTrainerDryRun: + """Tests for SFTTrainer.train(dry_run=True).""" + + @patch('sagemaker.train.sft_trainer._validate_and_resolve_model_package_group') + @patch('sagemaker.train.sft_trainer._get_fine_tuning_options_and_model_arn') + @patch('sagemaker.train.sft_trainer.TrainDefaults.get_role') + @patch('sagemaker.train.sft_trainer.TrainDefaults.get_sagemaker_session') + @patch('sagemaker.train.sft_trainer._get_unique_name') + @patch('sagemaker.train.sft_trainer._create_input_data_config') + @patch('sagemaker.train.sft_trainer._convert_input_data_to_channels') + @patch('sagemaker.train.sft_trainer._create_output_config') + @patch('sagemaker.train.sft_trainer._create_serverless_config') + @patch('sagemaker.train.sft_trainer._create_mlflow_config') + @patch('sagemaker.train.sft_trainer._create_model_package_config') + @patch('sagemaker.train.sft_trainer._validate_hyperparameter_values') + @patch('sagemaker.core.resources.TrainingJob.create') + @patch('sagemaker.train.common_utils.data_utils.validate_data_path_exists') + def test_dry_run_returns_none_without_submitting( + self, mock_validate_s3, mock_create, mock_validate_hp, mock_model_pkg, + mock_mlflow, mock_serverless, mock_output, mock_channels, mock_input, + mock_name, mock_session, mock_role, mock_options, mock_group, + ): + mock_group.return_value = "test-group" + mock_hp = Mock() + mock_hp.to_dict.return_value = {"lr": "0.001"} + mock_hp._specs = {} + mock_options.return_value = (mock_hp, "model-arn", False) + + sess = Mock() + sess.boto_session.region_name = "us-east-1" + mock_session.return_value = sess + mock_role.return_value = "test-role" + mock_name.return_value = "job-name" + mock_input.return_value = [Mock()] + mock_channels.return_value = [Mock()] + mock_output.return_value = Mock() + mock_serverless.return_value = Mock() + mock_mlflow.return_value = Mock() + mock_model_pkg.return_value = Mock() + + trainer = SFTTrainer( + model="test-model", model_package_group="test-group", + training_dataset="s3://bucket/train.jsonl", + ) + trainer.train(dry_run=True) + + mock_create.assert_not_called() + # Existing validation still ran + mock_role.assert_called_once() + mock_validate_hp.assert_called_once() + + @patch('sagemaker.train.sft_trainer._validate_and_resolve_model_package_group') + @patch('sagemaker.train.sft_trainer._get_fine_tuning_options_and_model_arn') + @patch('sagemaker.train.sft_trainer.TrainDefaults.get_sagemaker_session') + @patch('sagemaker.train.sft_trainer.TrainDefaults.get_role') + def test_dry_run_raises_on_role_validation_failure( + self, mock_role, mock_session, mock_options, mock_group, + ): + mock_group.return_value = "test-group" + mock_hp = Mock() + mock_hp.to_dict.return_value = {} + mock_hp._specs = {} + mock_options.return_value = (mock_hp, "model-arn", False) + + sess = Mock() + sess.boto_session.region_name = "us-east-1" + mock_session.return_value = sess + mock_role.side_effect = ValueError("Missing permissions") + + trainer = SFTTrainer( + model="test-model", model_package_group="test-group", + training_dataset="s3://bucket/train.jsonl", + ) + + with pytest.raises(ValueError, match="Missing permissions"): + trainer.train(dry_run=True) diff --git a/v3-examples/model-customization-examples/benchmark_demo.ipynb b/v3-examples/model-customization-examples/benchmark_demo.ipynb index 557e01ec35..1a48a3ff07 100644 --- a/v3-examples/model-customization-examples/benchmark_demo.ipynb +++ b/v3-examples/model-customization-examples/benchmark_demo.ipynb @@ -194,6 +194,28 @@ "- AssociateLineage: Links evaluation results to lineage tracking" ] }, + { + "cell_type": "markdown", + "id": "dry-run-explanation", + "metadata": {}, + "source": [ + "### Dry Run (Validation Only)\n", + "\n", + "Use `dry_run=True` to validate IAM permissions and model resolution without submitting an evaluation." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dry-run-validate", + "metadata": {}, + "outputs": [], + "source": [ + "# Validate configuration without launching a training job\n", + "evaluator.train(dry_run=True)\n", + "print(\"Dry-run passed — configuration is valid.\")" + ] + }, { "cell_type": "code", "metadata": {}, diff --git a/v3-examples/model-customization-examples/custom_scorer_demo.ipynb b/v3-examples/model-customization-examples/custom_scorer_demo.ipynb index c3706c9cf9..a930d17961 100644 --- a/v3-examples/model-customization-examples/custom_scorer_demo.ipynb +++ b/v3-examples/model-customization-examples/custom_scorer_demo.ipynb @@ -183,6 +183,28 @@ "3. Return an `EvaluationPipelineExecution` object for monitoring" ] }, + { + "cell_type": "markdown", + "id": "dry-run-explanation", + "metadata": {}, + "source": [ + "### Dry Run (Validation Only)\n", + "\n", + "Use `dry_run=True` to validate IAM permissions, model resolution, and dataset paths without submitting an evaluation." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dry-run-validate", + "metadata": {}, + "outputs": [], + "source": [ + "# Validate configuration without launching a training job\n", + "evaluator.train(dry_run=True)\n", + "print(\"Dry-run passed — configuration is valid.\")" + ] + }, { "cell_type": "code", "metadata": {}, diff --git a/v3-examples/model-customization-examples/dpo_trainer_example_notebook_v3_prod.ipynb b/v3-examples/model-customization-examples/dpo_trainer_example_notebook_v3_prod.ipynb index 6666c75467..2f28c83103 100644 --- a/v3-examples/model-customization-examples/dpo_trainer_example_notebook_v3_prod.ipynb +++ b/v3-examples/model-customization-examples/dpo_trainer_example_notebook_v3_prod.ipynb @@ -142,6 +142,28 @@ "\n" ] }, + { + "cell_type": "markdown", + "id": "dry-run-explanation", + "metadata": {}, + "source": [ + "### Dry Run (Validation Only)\n", + "\n", + "Use `dry_run=True` to validate IAM permissions, data paths, hyperparameters, infrastructure without submitting a job." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dry-run-validate", + "metadata": {}, + "outputs": [], + "source": [ + "# Validate configuration without launching a training job\n", + "trainer.train(dry_run=True)\n", + "print(\"Dry-run passed — configuration is valid.\")" + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/v3-examples/model-customization-examples/llm_as_judge_demo.ipynb b/v3-examples/model-customization-examples/llm_as_judge_demo.ipynb index 54f30d3adb..3e4a28c4ba 100644 --- a/v3-examples/model-customization-examples/llm_as_judge_demo.ipynb +++ b/v3-examples/model-customization-examples/llm_as_judge_demo.ipynb @@ -291,6 +291,28 @@ "3. Use the judge model to evaluate responses with built-in and custom metrics" ] }, + { + "cell_type": "markdown", + "id": "dry-run-explanation", + "metadata": {}, + "source": [ + "### Dry Run (Validation Only)\n", + "\n", + "Use `dry_run=True` to validate IAM permissions, model resolution, and dataset paths without submitting an evaluation." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dry-run-validate", + "metadata": {}, + "outputs": [], + "source": [ + "# Validate configuration without launching an evaluation job\n", + "evaluator.evaluate(dry_run=True)\n", + "print(\"Dry-run passed \u2014 configuration is valid.\")" + ] + }, { "cell_type": "code", "metadata": {}, diff --git a/v3-examples/model-customization-examples/rlaif_finetuning_example_notebook_v3_prod.ipynb b/v3-examples/model-customization-examples/rlaif_finetuning_example_notebook_v3_prod.ipynb index 02f14c5c8a..80191595c1 100644 --- a/v3-examples/model-customization-examples/rlaif_finetuning_example_notebook_v3_prod.ipynb +++ b/v3-examples/model-customization-examples/rlaif_finetuning_example_notebook_v3_prod.ipynb @@ -200,6 +200,28 @@ "#### Start RLAIF training\n" ] }, + { + "cell_type": "markdown", + "id": "dry-run-explanation", + "metadata": {}, + "source": [ + "### Dry Run (Validation Only)\n", + "\n", + "Use `dry_run=True` to validate IAM permissions, data paths, hyperparameters, infrastructure without submitting a job." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dry-run-validate", + "metadata": {}, + "outputs": [], + "source": [ + "# Validate configuration without launching a training job\n", + "rlaif_trainer.train(dry_run=True)\n", + "print(\"Dry-run passed — configuration is valid.\")" + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/v3-examples/model-customization-examples/rlvr_finetuning_example_notebook_v3_prod.ipynb b/v3-examples/model-customization-examples/rlvr_finetuning_example_notebook_v3_prod.ipynb index 55911557dd..3cc378c614 100644 --- a/v3-examples/model-customization-examples/rlvr_finetuning_example_notebook_v3_prod.ipynb +++ b/v3-examples/model-customization-examples/rlvr_finetuning_example_notebook_v3_prod.ipynb @@ -186,6 +186,28 @@ "#### Start RLVR training\n" ] }, + { + "cell_type": "markdown", + "id": "dry-run-explanation", + "metadata": {}, + "source": [ + "### Dry Run (Validation Only)\n", + "\n", + "Use `dry_run=True` to validate IAM permissions, data paths, hyperparameters, infrastructure without submitting a job." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dry-run-validate", + "metadata": {}, + "outputs": [], + "source": [ + "# Validate configuration without launching a training job\n", + "rlvr_trainer.train(dry_run=True)\n", + "print(\"Dry-run passed — configuration is valid.\")" + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/v3-examples/model-customization-examples/sft_finetuning_example_notebook_pysdk_prod_v3.ipynb b/v3-examples/model-customization-examples/sft_finetuning_example_notebook_pysdk_prod_v3.ipynb index cc45e38c99..1013bf672c 100644 --- a/v3-examples/model-customization-examples/sft_finetuning_example_notebook_pysdk_prod_v3.ipynb +++ b/v3-examples/model-customization-examples/sft_finetuning_example_notebook_pysdk_prod_v3.ipynb @@ -218,6 +218,28 @@ "#### Start SFT training\n" ] }, + { + "cell_type": "markdown", + "id": "dry-run-explanation", + "metadata": {}, + "source": [ + "### Dry Run (Validation Only)\n", + "\n", + "Use `dry_run=True` to validate IAM permissions, data paths, hyperparameters, infrastructure without submitting a job." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dry-run-validate", + "metadata": {}, + "outputs": [], + "source": [ + "# Validate configuration without launching a training job\n", + "sft_trainer.train(dry_run=True)\n", + "print(\"Dry-run passed — configuration is valid.\")" + ] + }, { "cell_type": "code", "execution_count": null,