diff --git a/dashscope/cli/agentic_rl.py b/dashscope/cli/agentic_rl.py index cd8386c..f850f71 100644 --- a/dashscope/cli/agentic_rl.py +++ b/dashscope/cli/agentic_rl.py @@ -460,6 +460,14 @@ def run( "--config", help="Path to YAML configuration file", ), + teacher_model: Optional[str] = typer.Option( + None, + "--teacher-model", + help=( + "Enable OPD with this teacher model (overrides YAML); " + "Rollout and Reward are optional" + ), + ), job_name: Optional[str] = typer.Option( None, help="Custom name for the tuning job", @@ -491,10 +499,8 @@ def run( 1. Configuration-driven: Use -c/--config to specify a YAML file 2. Direct parameter: Provide all required arguments via CLI options - Required parameters: - - rollout_classpath - - reward_classpaths (at least one) - - training_files (at least one) + Rollout and Reward are required for regular reinforcement learning, but + optional for OPD. """ output_format = validate_output_format(output_format) _apply_verbose(verbose) @@ -505,6 +511,7 @@ def run( # Prepare workflow parameters run_kwargs = { + "teacher_model": teacher_model, "job_name": job_name, } diff --git a/dashscope/finetune/agentic_rl.py b/dashscope/finetune/agentic_rl.py index aed51af..6eb707a 100644 --- a/dashscope/finetune/agentic_rl.py +++ b/dashscope/finetune/agentic_rl.py @@ -28,6 +28,7 @@ from dashscope.finetune.reinforcement import ( FunctionType, DatasetsType, + TrainingType, ) from dashscope.finetune.reinforcement import ( RewardInput, @@ -171,11 +172,14 @@ def submit_job( hyper_parameters: Optional[Dict[str, str]] = None, resources: Optional[Dict[str, str]] = None, job_name: Optional[str] = None, + teacher_model: Optional[str] = None, **kwargs, ) -> FineTune: """ Submit RL tuning job to the platform. """ + teacher_model = teacher_model or self.tuning.teacher_model + # Resolve job name (fallback to class default) if job_name: self.tuning.name = job_name @@ -230,6 +234,15 @@ def submit_job( if resources: self.tuning.training.resources = resources + training_type = self.tuning.training.type + if teacher_model: + training_type = TrainingType.PG_OPD + elif training_type == TrainingType.PG_OPD: + raise ValueErrorWithCode( + f"teacher_model is required when training.type is {training_type}", + error_code=3006, + ) + request = { "model": self.tuning.model.name, "training_datasets": [ds.model_dump() for ds in training_datasets], @@ -240,7 +253,8 @@ def submit_job( "rewards": rewards, "hyper_parameters": self.tuning.training.hyper_parameters, "resource_config": self.tuning.training.resources, - "training_type": str(self.tuning.training.type), + "training_type": str(training_type), + "teacher_model": teacher_model, "job_name": job_name_with_suffix, } request = deep_remove_none(request) @@ -287,6 +301,7 @@ async def run( hyper_parameters: Optional[Dict[str, str]] = None, resources: Optional[Dict[str, str]] = None, job_name: Optional[str] = None, + teacher_model: Optional[str] = None, **kwargs, ) -> FineTune: """ @@ -315,6 +330,7 @@ async def run( hyper_parameters=hyper_parameters, resources=resources, job_name=job_name, + teacher_model=teacher_model, **kwargs, ) except Exception as e: diff --git a/dashscope/finetune/reinforcement/common/model.py b/dashscope/finetune/reinforcement/common/model.py index 7819adf..d0eb5a2 100644 --- a/dashscope/finetune/reinforcement/common/model.py +++ b/dashscope/finetune/reinforcement/common/model.py @@ -1397,6 +1397,7 @@ class TuningModel(Models, BaseModel): name: str = Field(default="agentic-rl", min_length=1, max_length=256) model: FoundationModel = FoundationModel() + teacher_model: Optional[str] = None functions: List[FunctionComponentUnion] = Field(default_factory=list) datasets: List[Dataset] = Field(default_factory=list) training: Training = Training() diff --git a/dashscope/finetune/reinforcement/common/model_types.py b/dashscope/finetune/reinforcement/common/model_types.py index 4796aaf..025b8f0 100644 --- a/dashscope/finetune/reinforcement/common/model_types.py +++ b/dashscope/finetune/reinforcement/common/model_types.py @@ -47,6 +47,7 @@ def __str__(self): class TrainingType(str, Enum): TRAINING_TYPE = "reinforcement" + PG_OPD = "pg_opd" def __str__(self): return self.value diff --git a/dashscope/finetune/reinforcement/common/utils.py b/dashscope/finetune/reinforcement/common/utils.py index ff3ea45..a3d58a5 100644 --- a/dashscope/finetune/reinforcement/common/utils.py +++ b/dashscope/finetune/reinforcement/common/utils.py @@ -381,14 +381,15 @@ def create_deployment_files( function_layer_used=FC_LAYER_USED, ) - with open(FC_FILES_START, "w", encoding="utf-8") as f: + start_file = os.path.join(dirpath, FC_FILES_START) + with open(start_file, "w", encoding="utf-8") as f: f.write(content) # Add execute permission on Unix systems if os.name == "posix": - os.chmod(FC_FILES_START, 0o755) + os.chmod(start_file, 0o755) - logger.debug(f"Generated startup script: {FC_FILES_START}") + logger.debug(f"Generated startup script: {start_file}") except Exception as e: raise RuntimeErrorWithCode( "Deployment file creation error", diff --git a/dashscope/finetune/reinforcement/examples/workspace/README-zh.md b/dashscope/finetune/reinforcement/examples/workspace/README-zh.md index f40f438..7f3d9cc 100644 --- a/dashscope/finetune/reinforcement/examples/workspace/README-zh.md +++ b/dashscope/finetune/reinforcement/examples/workspace/README-zh.md @@ -56,6 +56,16 @@ langchain-openai==1.2.0 > 注：functions/目录下需要包含__init__.py文件 +函数组件是否必需取决于训练配置： + +| 配置 | 场景 | 自定义 Rollout | 自定义 Reward | +|---|---|---:|---:| +| `rl-job.yaml` | 普通强化学习 | 必选 | 至少一个 | +| `opd-job.yaml` 不保留函数块 | 仅 Teacher | 否 | 否 | +| `opd-job.yaml` 仅保留 Reward | Teacher + Reward | 否 | 是 | +| `opd-job.yaml` 仅保留 Rollout | Teacher + Rollout | 是 | 否 | +| `opd-job.yaml` 保留两个函数块 | Teacher + Rollout + Reward | 是 | 是 | + ### 3.2 准备训练数据 在`data`目录下添加数据集文件： @@ -67,19 +77,56 @@ langchain-openai==1.2.0 ### 4.1 函数执行（注册+测试） +普通强化学习必须注册自定义 Rollout 和至少一个 Reward。OPD 的 +Rollout、Reward 可选。 + ```bash python test_functions.py ``` ### 4.2 工作流执行（YAML配置+生命周期管理） +普通强化学习与 OPD 使用完全相同的 SDK 工作流，只需选择对应 YAML： + +```python +from dashscope.finetune.agentic_rl import AgenticRL + +client = AgenticRL() +# 普通强化学习使用 rl-job.yaml；OPD 使用 opd-job.yaml +client.init(config_path="rl-job.yaml") +result = await client.run() +``` + +`opd-job.yaml` 默认保留两个函数块，表示 Teacher + Rollout + Reward。删除 +Reward 块表示 Teacher + Rollout；删除 Rollout 块表示 Teacher + Reward； +两个都删除表示仅 Teacher。 + ```bash +# submit_job.py 默认使用普通强化学习 rl-job.yaml python submit_job.py ``` ## 5. 使用CLI执行任务 示例代码：cli.sh +`cli.sh` 展示普通强化学习的完整流程。执行 OPD 时，对 `opd-job.yaml` 中已经 +删除的函数块跳过对应注册和测试；数据提交、`rl run` 和任务生命周期命令 +保持不变。 + +CLI 与 SDK 使用同一份 YAML： + +```bash +# 普通强化学习 +dashscope rl run -c rl-job.yaml + +# OPD +dashscope rl run -c opd-job.yaml + +# 覆盖 opd-job.yaml 中配置的 Teacher 模型 +dashscope rl run -c opd-job.yaml \ + --teacher-model qwen3.5-397b-a17b +``` + ```bash dashscope rl --help # 查看完整命令帮助 diff --git a/dashscope/finetune/reinforcement/examples/workspace/README.md b/dashscope/finetune/reinforcement/examples/workspace/README.md index f4bf78a..02db62f 100644 --- a/dashscope/finetune/reinforcement/examples/workspace/README.md +++ b/dashscope/finetune/reinforcement/examples/workspace/README.md @@ -56,6 +56,16 @@ Develop functions under the `functions` directory: > Note: The `functions/` directory must contain an `__init__.py` file +Function components depend on the selected training configuration: + +| Configuration | Mode | Custom Rollout | Custom Reward | +|---|---|---:|---:| +| `rl-job.yaml` | Regular reinforcement | Required | At least one | +| `opd-job.yaml` without function blocks | Teacher only | No | No | +| `opd-job.yaml` with Reward only | Teacher + Reward | No | Yes | +| `opd-job.yaml` with Rollout only | Teacher + Rollout | Yes | No | +| `opd-job.yaml` with both blocks | Teacher + Rollout + Reward | Yes | Yes | + ### 3.2 Preparing Training Data Add dataset files under the `data` directory: @@ -67,19 +77,57 @@ Add dataset files under the `data` directory: ### 4.1 Function Execution (Register + Test) +Regular reinforcement training requires a custom Rollout and at least one +Reward. Rollout and Reward are optional for OPD. + ```bash python test_functions.py ``` ### 4.2 Workflow Execution (YAML Config + Lifecycle Management) +Regular reinforcement training and OPD use the same SDK workflow. Select the +corresponding YAML: + +```python +from dashscope.finetune.agentic_rl import AgenticRL + +client = AgenticRL() +# Use rl-job.yaml for regular reinforcement or opd-job.yaml for OPD +client.init(config_path="rl-job.yaml") +result = await client.run() +``` + +`opd-job.yaml` keeps both function blocks by default, representing Teacher + +Rollout + Reward. Remove the Reward block for Teacher + Rollout, remove the +Rollout block for Teacher + Reward, or remove both for Teacher only. + ```bash +# submit_job.py uses the regular reinforcement rl-job.yaml by default python submit_job.py ``` ## 5. Executing Tasks with CLI Example code：cli.sh +`cli.sh` demonstrates the regular reinforcement workflow. For OPD, skip +registration and testing for function blocks removed from `opd-job.yaml`; dataset +submission, `rl run`, and job lifecycle commands remain unchanged. + +The CLI consumes the same YAML files as the SDK: + +```bash +# Regular reinforcement +dashscope rl run -c rl-job.yaml + +# OPD +dashscope rl run -c opd-job.yaml + +# Override the Teacher configured in opd-job.yaml +dashscope rl run -c opd-job.yaml \ + --teacher-model qwen3.5-397b-a17b +``` + ```bash dashscope rl --help # View full command help diff --git a/dashscope/finetune/reinforcement/examples/workspace/UserGuide-zh.md b/dashscope/finetune/reinforcement/examples/workspace/UserGuide-zh.md index e620843..7d5b295 100644 --- a/dashscope/finetune/reinforcement/examples/workspace/UserGuide-zh.md +++ b/dashscope/finetune/reinforcement/examples/workspace/UserGuide-zh.md @@ -11,8 +11,9 @@ SDK 包含两个核心模块： 2. **调优模块**：处理数据集管理、超参数配置、作业提交和生命周期管理（状态、日志、取消）。 **工作流**： -1. 首先，在函数模块中注册函数（Rollout/Reward/Group Reward）。这些函数可以在本地或远程测试。 -2. 然后，使用调优模块上传数据集、配置作业并提交训练。 +1. 注册自定义函数。普通强化学习必须配置 Rollout 和至少一个 Reward； + OPD 的 Rollout、Reward 可选。 +2. 使用调优模块上传数据集、配置作业并提交训练。 --- @@ -45,7 +46,7 @@ workspace/ ├── data/ # 数据集 │ ├── calc_train_min.jsonl │ └── calc_validation_min.jsonl -├── functions/ # 自定义逻辑 +├── functions/ # 自定义函数；仅 OPD 可省略 Rollout/Reward │ ├── reward/ │ │ ├── group_reward_.py │ │ ├── reward.py @@ -54,9 +55,13 @@ workspace/ │ ├── rollout_only.py │ └── rollout.py ├── requirements.txt # 函数组件依赖项 -└── job.yaml # 可选：作业配置 +├── rl-job.yaml # 普通 Agentic RL 配置 +└── opd-job.yaml # OPD 配置 ``` +普通强化学习必须配置自定义 Rollout 和至少一个 Reward。只有 OPD 可以不配置 +Rollout、Reward，或者按需只配置其中一个。 + ### 2.5 依赖包（requirements.txt 指南） 此文件对于将函数组件部署到云端是**必需的**，必须位于工作空间根目录。 @@ -271,7 +276,14 @@ class MyRolloutProcessor(AbstractRolloutProcessor): * [CLI] 入口点: dashscope rl ### 4.1 作业配置 -使用 YAML 或代码初始化客户端。代码参数会覆盖 YAML 配置。 + +普通强化学习和 OPD 使用同一套 SDK/CLI 工作流，区别在于传入的 YAML 配置。 +代码参数会覆盖 YAML 配置。 + +| 训练方式 | 配置文件 | 自定义函数 | +|---|---|---| +| 普通强化学习 | `rl-job.yaml` | Rollout 必选，Reward 至少一个 | +| OPD | `opd-job.yaml` | Rollout、Reward 独立可选，可以都不配置 | **[SDK] \_\_init\_\_** ```python @@ -303,7 +315,45 @@ def init(self, config_path: Optional[str] = None, **kwargs) -> Self: ... **示例**: ```python from dashscope.finetune.agentic_rl import AgenticRL -rl = AgenticRL().init("job.yaml", job_name="custom_job") +config_path = "opd-job.yaml" # 普通强化学习使用 "rl-job.yaml" +rl = AgenticRL().init(config_path, job_name="custom_job") +result = await rl.run() +``` + +选择 `opd-job.yaml` 时，可以通过保留或删除 Rollout、Reward 函数块选择 +需要的能力： + +| 保留的函数块 | OPD 场景 | 轨迹来源 | 任务奖励 | +|---|---|---|---| +| 都不保留 | 仅 Teacher | 平台生成 | 不启用 | +| 仅 Reward | Teacher + Reward | 平台生成 | 自定义 Reward | +| 仅 Rollout | Teacher + Rollout | 自定义 Rollout | 不启用 | +| Rollout 和 Reward | Teacher + Rollout + Reward | 自定义 Rollout | 自定义 Reward | + +仓库提交的 `opd-job.yaml` 默认保留两个函数块。删除其中一个或两个，即可 +切换为其他 OPD 组合： + +```yaml +teacher_model: qwen3.5-397b-a17b + +functions: +# 删除此块后使用平台生成。 +- type: rollout + # ... +# 删除此块后不使用自定义任务奖励。 +- type: reward + # ... + +training: + type: pg_opd +``` + +```bash +# 普通强化学习 +dashscope rl run -c rl-job.yaml + +# OPD +dashscope rl run -c opd-job.yaml ``` ### 4.2 注册函数 @@ -455,18 +505,32 @@ dashscope rl test_functions "ro-ins--xxx" \ 自动注册函数、上传数据并提交作业。 ```python -def run(self, model: Optional[str] = None, training_files: Optional[Union[List[str], str]] = None, validation_files: Optional[Union[List[str], str]] = None, functions: Optional[Union[List[Union[RolloutFunctionComponent, RewardFunctionComponent, AgenticRLFunctionComponent]], RolloutFunctionComponent, RewardFunctionComponent, AgenticRLFunctionComponent]] = None, hyper_parameters: Optional[Dict[str, str]] = None, job_name: Optional[str] = None, workspace_dir: str = "./", **kwargs) -> FineTune: ... +async def run( + self, + model: Optional[str] = None, + training_datasets: Optional[List[TrainingDataset]] = None, + validation_datasets: Optional[List[ValidationDataset]] = None, + functions: Optional[ + Union[List[AgenticRLFunctionComponent], AgenticRLFunctionComponent] + ] = None, + hyper_parameters: Optional[Dict[str, str]] = None, + resources: Optional[Dict[str, str]] = None, + job_name: Optional[str] = None, + teacher_model: Optional[str] = None, + **kwargs, +) -> FineTune: ... ``` 完整的工作流执行（注册 + 上传 + 提交）。 **参数**: - `model`: 基础模型名称 -- `training_files`: 训练数据集文件 -- `validation_files`: 验证数据集文件 +- `training_datasets`: 训练数据集对象 +- `validation_datasets`: 验证数据集对象 - `functions`: 函数组件 - `hyper_parameters`: 训练超参数 +- `resources`: 训练资源配置 - `job_name`: 自定义作业名称 -- `workspace_dir`: 工作目录 +- `teacher_model`: Teacher 模型；设置后启用 OPD，Rollout 和 Reward 仍可省略 **返回**: `FineTune` 作业对象 @@ -521,7 +585,7 @@ job = await rl.run( **[CLI] run** -**用法: dashscope run [OPTIONS]** +**用法: dashscope rl run [OPTIONS]** ```bash 🚀 启动完整的 RL 调优工作流（函数注册 → 数据集上传 → 作业提交） @@ -530,13 +594,13 @@ job = await rl.run( 2. 直接参数: 通过 CLI 选项提供所有必需参数 必需参数: - - rollout_classpath - - reward_classpaths (至少一个) - - training_files (至少一个) + - training_files（至少一个） + - 强化学习需要配置 Rollout 和 Reward，OPD 中两者均为可选。 ╭─ 选项 ────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ │ --config -c PATH YAML 配置文件路径 │ │ --model TEXT 基础模型标识符 │ +│ --teacher-model TEXT 启用 OPD 并覆盖 YAML 中的 Teacher 模型；Rollout 和 Reward 可省略 │ │ --training-files TEXT 训练数据集文件路径 │ │ --validation-files TEXT 验证数据集文件路径 │ │ --rollout-classpath TEXT Rollout 类的 Python 导入路径 (module:Class) │ @@ -565,7 +629,7 @@ job = await rl.run( **示例**: 运行完整工作流（自动） ```bash dashscope rl run \ - --config "job.yaml" \ + --config "rl-job.yaml" \ --verbose ``` diff --git a/dashscope/finetune/reinforcement/examples/workspace/UserGuide.md b/dashscope/finetune/reinforcement/examples/workspace/UserGuide.md index 7e8aa0d..aded12a 100644 --- a/dashscope/finetune/reinforcement/examples/workspace/UserGuide.md +++ b/dashscope/finetune/reinforcement/examples/workspace/UserGuide.md @@ -10,10 +10,11 @@ The SDK consists of two core modules: 1. **Functions Module**: Manages custom Python code for **Rollout** (trajectory generation), **Reward** (scoring), and **Group Reward** (batch scoring). It supports automatic registration, testing, and built-in **Observability (Tracing)**. 2. **Tuning Module**: Handles dataset management, hyperparameter configuration, job submission, and lifecycle management (status, logs, cancellation). -``` **Workflow**: -1. First, register functions (Rollout/Reward/Group Reward) in the Functions Module. These can be tested locally or remotely. -2. Then, use the Tuning Module to upload datasets, configure the job, and submit it for training. +1. Register custom functions. Regular reinforcement training requires a + Rollout and at least one Reward; Rollout and Reward are optional for OPD. +2. Use the Tuning Module to upload datasets, configure the job, and submit it + for training. --- @@ -46,16 +47,21 @@ workspace/ ├── data/ # Datasets │ ├── training.jsonl │ └── validation.jsonl -├── functions/ # Custom Logic +├── functions/ # Custom functions; optional only for OPD │ ├── reward/ │ │ ├── group_reward.py │ │ └── reward.py │ └── rollout/ │ └── rollout.py ├── requirements.txt # Dependencies for Function Components -└── job.yaml # Optional: Job configuration +├── rl-job.yaml # Regular Agentic RL configuration +└── opd-job.yaml # OPD configuration ``` +Regular reinforcement training requires a custom Rollout and at least one +Reward. Only OPD may omit Rollout and Reward, or configure either one +independently. + ### 2.5 Dependency Packages (requirements.txt Guidelines) This file is **mandatory** for deploying Function Components to the cloud. It must reside in the workspace root. @@ -259,7 +265,14 @@ class MyRolloutProcessor(AbstractRolloutProcessor): * [CLI] Entry point: dashscope rl ### 4.1 Job Configuration -Initialize the client using YAML or code. Code arguments override YAML. + +Regular reinforcement training and OPD use the same SDK/CLI workflow. Select +the corresponding YAML configuration; code arguments override YAML values. + +| Training type | Configuration | Custom functions | +|---|---|---| +| Regular reinforcement | `rl-job.yaml` | Rollout is required, with at least one Reward | +| OPD | `opd-job.yaml` | Rollout and Reward are independently optional and may both be omitted | **[SDK] \_\_init\_\_** ```python @@ -291,7 +304,45 @@ Initializes the instance from a YAML configuration file. **Example**: ```python from dashscope.finetune.agentic_rl import AgenticRL -rl = AgenticRL().init("job.yaml", job_name="custom_job") +config_path = "opd-job.yaml" # Use "rl-job.yaml" for regular reinforcement +rl = AgenticRL().init(config_path, job_name="custom_job") +result = await rl.run() +``` + +When `opd-job.yaml` is selected, keep or remove the Rollout and Reward blocks +to choose the required capabilities: + +| Function blocks kept | OPD mode | Trajectory source | Task reward | +|---|---|---|---| +| Neither | Teacher only | Platform | Disabled | +| Reward only | Teacher + Reward | Platform | Custom Reward | +| Rollout only | Teacher + Rollout | Custom Rollout | Disabled | +| Rollout and Reward | Teacher + Rollout + Reward | Custom Rollout | Custom Reward | + +The committed `opd-job.yaml` keeps both blocks. Delete either block, or both, +to select another OPD combination: + +```yaml +teacher_model: qwen3.5-397b-a17b + +functions: +# Remove this block to use platform generation. +- type: rollout + # ... +# Remove this block to disable custom task rewards. +- type: reward + # ... + +training: + type: pg_opd +``` + +```bash +# Regular reinforcement +dashscope rl run -c rl-job.yaml + +# OPD +dashscope rl run -c opd-job.yaml ``` ### 4.2 Registering Functions @@ -444,18 +495,32 @@ Automatically registers functions, uploads data, and submits the job. **[SDK] [run](submit_job.py)** ```python -def run(self, model: Optional[str] = None, training_files: Optional[Union[List[str], str]] = None, validation_files: Optional[Union[List[str], str]] = None, functions: Optional[Union[List[Union[RolloutFunctionComponent, RewardFunctionComponent, AgenticRLFunctionComponent]], RolloutFunctionComponent, RewardFunctionComponent, AgenticRLFunctionComponent]] = None, hyper_parameters: Optional[Dict[str, str]] = None, job_name: Optional[str] = None, workspace_dir: str = "./", **kwargs) -> FineTune: ... +async def run( + self, + model: Optional[str] = None, + training_datasets: Optional[List[TrainingDataset]] = None, + validation_datasets: Optional[List[ValidationDataset]] = None, + functions: Optional[ + Union[List[AgenticRLFunctionComponent], AgenticRLFunctionComponent] + ] = None, + hyper_parameters: Optional[Dict[str, str]] = None, + resources: Optional[Dict[str, str]] = None, + job_name: Optional[str] = None, + teacher_model: Optional[str] = None, + **kwargs, +) -> FineTune: ... ``` Full workflow execution (registration + upload + submission). **Parameters**: - `model`: Base model name -- `training_files`: Training dataset files -- `validation_files`: Validation dataset files +- `training_datasets`: Training dataset objects +- `validation_datasets`: Validation dataset objects - `functions`: Function components - `hyper_parameters`: Training hyper_parameters +- `resources`: Training resource configuration - `job_name`: Custom job name -- `workspace_dir`: Working directory +- `teacher_model`: Teacher model; enables OPD while Rollout and Reward remain optional **Returns**: `FineTune` job object @@ -510,7 +575,7 @@ job = await rl.run( **[CLI] run** -**Usage: dashscope run [OPTIONS]** +**Usage: dashscope rl run [OPTIONS]** ```bash 🚀 Launch the complete RL tuning workflow (function registration → dataset upload → job submission) @@ -519,13 +584,13 @@ job = await rl.run( 2. Direct parameter: Provide all required arguments via CLI options Required parameters: - - rollout_classpath - - reward_classpaths (at least one) - training_files (at least one) + - Rollout and Reward are required for reinforcement learning, but optional for OPD. ╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ │ --config -c PATH Path to YAML configuration file │ │ --model TEXT Base model identifier │ +│ --teacher-model TEXT Enable OPD and override the Teacher model in YAML; Rollout and Reward are optional │ │ --training-files TEXT Paths to training dataset files │ │ --validation-files TEXT Paths to validation dataset files │ │ --rollout-classpath TEXT Python import path to rollout class (module:Class) │ @@ -554,7 +619,7 @@ job = await rl.run( **Example**: Run Full Workflow (Auto) ```bash dashscope rl run \ - --config "job.yaml" \ + --config "rl-job.yaml" \ --verbose ``` diff --git a/dashscope/finetune/reinforcement/examples/workspace/cli.sh b/dashscope/finetune/reinforcement/examples/workspace/cli.sh index f9ca679..39063e5 100644 --- a/dashscope/finetune/reinforcement/examples/workspace/cli.sh +++ b/dashscope/finetune/reinforcement/examples/workspace/cli.sh @@ -87,7 +87,7 @@ # - Operation Modes: # a. Config-Driven Mode: # ```bash -# dashscope rl run -c job.yaml -o json +# dashscope rl run -c rl-job.yaml -o json # ``` # - Key Options: # * -c/--config: YAML configuration file path @@ -177,7 +177,7 @@ echo "$UPLOAD_RESULT" # ===================== 4. Submit job ===================== echo ">>> Step 4: Submitting job..." -RUN_RESULT=$(dashscope rl run -c job.yaml -o json) +RUN_RESULT=$(dashscope rl run -c rl-job.yaml -o json) echo "$RUN_RESULT" JOB_ID=$(echo "$RUN_RESULT" | json_get "['job_id']") diff --git a/dashscope/finetune/reinforcement/examples/workspace/opd-job.yaml b/dashscope/finetune/reinforcement/examples/workspace/opd-job.yaml new file mode 100644 index 0000000..f1c4e69 --- /dev/null +++ b/dashscope/finetune/reinforcement/examples/workspace/opd-job.yaml @@ -0,0 +1,70 @@ +# OPD capability combinations: +# - Remove both function blocks: Teacher only. +# - Keep only the reward block: Teacher + Reward. +# - Keep only the rollout block: Teacher + Rollout. +# - Keep both blocks (the current configuration): Teacher + Rollout + Reward. +name: opd-from-yaml +model: + name: qwen3.5-9b +teacher_model: qwen3.5-397b-a17b +functions: +# Optional OPD capability: remove this block to use platform generation. +- type: rollout + name: rollout-1 + timeout: 600 + fcmodel: + zipdir: ./ + classpath: functions.rollout.rollout.CalcXRolloutProcessor + requirements_path: ./requirements.txt + runtime: + cpu: 2.0 + memory_size: 4096 + disk_size: 512 + concurrency: 30 + capacity: 30 + max_capacity: 60 + min_capacity: 30 + memory_scale_threshold: 0.6 + concurrency_scale_threshold: 0.6 + env: {} +# Optional OPD capability: remove this block to disable custom task rewards. +- type: reward + name: rewards-1 + timeout: 120 + fcmodel: + zipdir: ./ + classpath: functions.reward.reward.DemoRewardProcessor + requirements_path: ./requirements.txt + runtime: + cpu: 2.0 + memory_size: 4096 + disk_size: 512 + concurrency: 30 + capacity: 30 + max_capacity: 60 + min_capacity: 30 + memory_scale_threshold: 0.6 + concurrency_scale_threshold: 0.6 + env: {} + weight: 1.0 + reward_metric_weight: + reward_metric_weightA: 0.3 + reward_metric_weightB: 0.7 +datasets: +- type: training + data_source_type: file_id + file_name: ./data/calc_train_min.jsonl +- type: validation + data_source_type: file_id + file_name: ./data/calc_validation_min.jsonl +training: + type: pg_opd + hyper_parameters: + eval_steps: 1 + kl_loss_coef: 0.002 + learning_rate: 2e-6 + lr_scheduler_type: cosine + n_epochs: 1 + n_rollouts: 8 + ppo_mini_batch_size: 8 +observability: {} diff --git a/dashscope/finetune/reinforcement/examples/workspace/job.yaml b/dashscope/finetune/reinforcement/examples/workspace/rl-job.yaml similarity index 100% rename from dashscope/finetune/reinforcement/examples/workspace/job.yaml rename to dashscope/finetune/reinforcement/examples/workspace/rl-job.yaml diff --git a/dashscope/finetune/reinforcement/examples/workspace/submit_job.py b/dashscope/finetune/reinforcement/examples/workspace/submit_job.py index 4ea37bb..c4b971c 100644 --- a/dashscope/finetune/reinforcement/examples/workspace/submit_job.py +++ b/dashscope/finetune/reinforcement/examples/workspace/submit_job.py @@ -22,9 +22,9 @@ - Initializes the RL workflow using a YAML configuration file - Key Parameters: * config_path (required): Path to YAML configuration file (e.g., - "job.yaml") + "rl-job.yaml") * job_name: Custom identifier for the training job - - Configuration File Example (job.yaml) + - Configuration File Example (rl-job.yaml) - Benefits: * Decouples configuration from code * Enables version-controlled parameter management @@ -45,7 +45,7 @@ """ # Example YAML-driven usage: -# client.init(config_path="job.yaml", job_name="agentic-rl") +# client.init(config_path="rl-job.yaml", job_name="agentic-rl") # await client.run() # Uses YAML configuration from dashscope.finetune.agentic_rl import AgenticRL @@ -201,7 +201,7 @@ async def main_workflow_yaml(): client = AgenticRL() client.init( - config_path="job.yaml", + config_path="rl-job.yaml", name="agentic-rl-from-yaml", ) client.tuning.to_yaml(file_path="init.yaml") @@ -213,7 +213,7 @@ async def main_workflow_yaml(): if result.status_code == 200: job_id = result.output.job_id logger.info( - f"agentic rl(from job.yaml) submit: {job_id=}, {result=}", + f"agentic rl(from rl-job.yaml) submit: {job_id=}, {result=}", ) else: raise ValueError(f"agentic rl submit: {result}") diff --git a/setup.py b/setup.py index 791a115..1d249dd 100644 --- a/setup.py +++ b/setup.py @@ -48,7 +48,7 @@ def readme(): license="Apache 2.0", url=url, packages=setuptools.find_packages( - exclude=("tests"), + exclude=("tests", "tests.*"), ), # pylint: disable=superfluous-parens classifiers=[ "Development Status :: 4 - Beta", diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit/test_agentic_rl_cli_opd.py b/tests/unit/test_agentic_rl_cli_opd.py new file mode 100644 index 0000000..e217842 --- /dev/null +++ b/tests/unit/test_agentic_rl_cli_opd.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- +"""Minimal CLI coverage for teacher_model.""" + +import re +from unittest.mock import patch + +import pytest +from typer.main import get_command +from typer.testing import CliRunner + +from dashscope.cli.agentic_rl import _run_workflow_async, app +from dashscope.finetune.agentic_rl import AgenticRL + + +def test_run_help_explains_conditional_function_requirements(): + result = CliRunner().invoke(app, ["run", "--help"]) + assert result.exit_code == 0 + + # 参数存在性走 typer 命令模型校验：渲染文本在不同终端宽度/平台下会被 + # rich 截断或连字符换行（CI Linux 曾因此误报 --teacher-model 缺失）， + # 不断言渲染后的 help 文本里出现完整 option 名。 + run_cmd = get_command(app).commands["run"] + assert "teacher_model" in {param.name for param in run_cmd.params} + + # help 文档仍须说明 OPD 的条件化函数要求；剥离 ANSI 后按空白归一， + # 只断言散文式描述（按空格折行，归一化后稳定）。 + help_text = " ".join(re.sub(r"\x1b\[[0-9;]*m", "", result.output).split()) + assert ( + "1. Configuration-driven: Use -c/--config to specify a YAML file" + in help_text + ) + assert ( + "2. Direct parameter: Provide all required arguments via CLI options" + in help_text + ) + assert ( + "Rollout and Reward are required for regular reinforcement learning, " + "but optional for OPD" in help_text + ) + + +@pytest.mark.asyncio +async def test_workflow_applies_teacher_model_option(tmp_path): + config = tmp_path / "opd.yaml" + config.write_text( + "name: opd-cli-test\n", + encoding="utf-8", + ) + + async def inspect_config(client): + return client.tuning.teacher_model + + with ( + patch( + "dashscope.finetune.agentic_rl.set_api_key", + ), + patch.object( + AgenticRL, + "run", + new=inspect_config, + ), + ): + result = await _run_workflow_async( + config_path=str(config), + api_key="test-api-key", + run_kwargs={"teacher_model": "qwen3.5-397b-a17b"}, + ) + + assert result == "qwen3.5-397b-a17b" diff --git a/tests/unit/test_agentic_rl_utils.py b/tests/unit/test_agentic_rl_utils.py index 2ea047c..defdbf3 100644 --- a/tests/unit/test_agentic_rl_utils.py +++ b/tests/unit/test_agentic_rl_utils.py @@ -108,16 +108,12 @@ def test_create_deployment_files(self, tmp_path): requirements_path="requirements.txt", ) - # Validate generated file (written to cwd by create_deployment_files) - try: - assert os.path.exists("start.sh") - with open("start.sh", "r", encoding="utf-8") as f: - content = f.read() - assert 'PROCESSOR_CLASS="processor.DemoProcessor"' in content - assert 'REQUIREMENTS_FILE="requirements.txt"' in content - finally: - if os.path.exists("start.sh"): - os.remove("start.sh") + # Validate generated file in the packaged workspace + start_file = test_dir / "start.sh" + assert start_file.exists() + content = start_file.read_text(encoding="utf-8") + assert 'PROCESSOR_CLASS="processor.DemoProcessor"' in content + assert 'REQUIREMENTS_FILE="requirements.txt"' in content def test_create_files_without_requirements(self, tmp_path): """Test creating deployment files without requirements.""" @@ -136,18 +132,12 @@ def test_create_files_without_requirements(self, tmp_path): requirements_path="", ) - # Validate generated file (written to cwd by create_deployment_files) - try: - assert os.path.exists("start.sh") - with open("start.sh", "r", encoding="utf-8") as f: - content = f.read() - assert ( - 'PROCESSOR_CLASS="module.processor.MyProcessor"' in content - ) - assert "local_packages=($SDK_PACKAGE)" in content - finally: - if os.path.exists("start.sh"): - os.remove("start.sh") + # Validate generated file in the packaged workspace + start_file = test_dir / "start.sh" + assert start_file.exists() + content = start_file.read_text(encoding="utf-8") + assert 'PROCESSOR_CLASS="module.processor.MyProcessor"' in content + assert "local_packages=($SDK_PACKAGE)" in content def test_invalid_file_path(self, tmp_path): """Test handling of invalid file paths.""" diff --git a/tests/unit/test_opd.py b/tests/unit/test_opd.py new file mode 100644 index 0000000..95748bf --- /dev/null +++ b/tests/unit/test_opd.py @@ -0,0 +1,251 @@ +# -*- coding: utf-8 -*- +"""Minimal OPD contract tests.""" +# pylint: disable=redefined-outer-name # pytest fixture 标准用法 + +from pathlib import Path +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from dashscope.client.base_api import CreateMixin +from dashscope.finetune.agentic_rl import AgenticRL +from dashscope.finetune.reinforcement import ( + DataSourceType, + RewardFunctionComponent, + RolloutFunctionComponent, + TrainingDataset, + TrainingType, +) +from dashscope.finetune.reinforcement.common.errors import ValueErrorWithCode + + +@pytest.fixture +def rl_client(): + with patch("dashscope.finetune.agentic_rl.set_api_key"): + return AgenticRL(api_key="test-key") + + +def _training_dataset(): + return TrainingDataset( + data_source_type=DataSourceType.FILE_ID, + file_id="file-train-001", + ) + + +def _success_response(): + return { + "status_code": 200, + "request_id": "request-001", + "output": { + "job_id": "ft-test-001", + "status": "PENDING", + }, + } + + +def test_pg_opd_training_type_is_public(): + assert TrainingType.PG_OPD == "pg_opd" + + +@pytest.mark.parametrize( + ("functions", "has_rollout", "reward_count"), + [ + (None, False, 0), + ([RewardFunctionComponent(entity_id="reward-001")], False, 1), + ([RolloutFunctionComponent(entity_id="rollout-001")], True, 0), + ( + [ + RolloutFunctionComponent(entity_id="rollout-001"), + RewardFunctionComponent(entity_id="reward-001"), + ], + True, + 1, + ), + ], +) +def test_teacher_model_enables_opd( + rl_client, + functions, + has_rollout, + reward_count, +): + with patch.object( + CreateMixin, + "call", + return_value=_success_response(), + ) as call: + rl_client.submit_job( + model="qwen3.5-9b", + datasets=[_training_dataset()], + functions=functions, + teacher_model="qwen3.5-397b-a17b", + ) + + request = call.call_args.args[0] + assert request["teacher_model"] == "qwen3.5-397b-a17b" + assert request["training_type"] == str(TrainingType.PG_OPD) + assert ("rollout" in request) is has_rollout + assert len(request.get("rewards", [])) == reward_count + + +def test_teacher_model_can_come_from_yaml(rl_client, tmp_path): + config = tmp_path / "opd.yaml" + config.write_text( + "teacher_model: qwen3.5-397b-a17b\n", + encoding="utf-8", + ) + rl_client.init(config_path=str(config)) + + with patch.object( + CreateMixin, + "call", + return_value=_success_response(), + ) as call: + rl_client.submit_job(datasets=[_training_dataset()]) + + request = call.call_args.args[0] + assert request["teacher_model"] == "qwen3.5-397b-a17b" + assert request["training_type"] == str(TrainingType.PG_OPD) + + +def test_opd_example_reuses_supported_rl_configuration(rl_client): + config = ( + Path(__file__).parents[2] + / "dashscope/finetune/reinforcement/examples/workspace" + / "opd-job.yaml" + ) + config_text = config.read_text(encoding="utf-8") + rl_client.init(config_path=str(config)) + + with patch.object( + CreateMixin, + "call", + return_value=_success_response(), + ) as call: + rl_client.submit_job() + + request = call.call_args.args[0] + assert request["teacher_model"] == "qwen3.5-397b-a17b" + assert request["training_type"] == str(TrainingType.PG_OPD) + assert rl_client.tuning.training.type == TrainingType.PG_OPD + assert request["hyper_parameters"] == { + "eval_steps": 1, + "kl_loss_coef": 0.002, + "learning_rate": "2e-6", + "lr_scheduler_type": "cosine", + "n_epochs": 1, + "n_rollouts": 8, + "ppo_mini_batch_size": 8, + } + assert rl_client.tuning.training.resources is None + assert request["training_datasets"][0]["file_name"] == ( + "./data/calc_train_min.jsonl" + ) + assert request["validation_datasets"][0]["file_name"] == ( + "./data/calc_validation_min.jsonl" + ) + assert request["training_datasets"][0]["data_source_type"] == "file_id" + assert "file_id" not in request["training_datasets"][0] + assert [type(function) for function in rl_client.tuning.functions] == [ + RolloutFunctionComponent, + RewardFunctionComponent, + ] + assert "Remove both function blocks: Teacher only." in config_text + assert "Keep only the reward block: Teacher + Reward." in config_text + assert "Keep only the rollout block: Teacher + Rollout." in config_text + assert "Keep both blocks" in config_text + + +@pytest.mark.asyncio +async def test_opd_example_uploads_local_datasets(rl_client): + config = ( + Path(__file__).parents[2] + / "dashscope/finetune/reinforcement/examples/workspace" + / "opd-job.yaml" + ) + rl_client.init(config_path=str(config)) + + with patch( + "dashscope.finetune.reinforcement.common.model.to_bailian_data", + new=AsyncMock( + side_effect=[["file-train-001"], ["file-validation-001"]], + ), + ): + train_ids, validation_ids = await rl_client.upload_datasets() + + assert train_ids == ["file-train-001"] + assert validation_ids == ["file-validation-001"] + assert rl_client.tuning.datasets[0].file_id == "file-train-001" + assert rl_client.tuning.datasets[1].file_id == "file-validation-001" + + +def test_regular_rl_example_keeps_resources_and_rl_hyperparameters(rl_client): + config = ( + Path(__file__).parents[2] + / "dashscope/finetune/reinforcement/examples/workspace" + / "rl-job.yaml" + ) + rl_client.init(config_path=str(config)) + + assert rl_client.tuning.teacher_model is None + assert rl_client.tuning.training.type == TrainingType.TRAINING_TYPE + assert rl_client.tuning.training.hyper_parameters["algorithm"] == "gspo" + assert ( + rl_client.tuning.training.hyper_parameters["lr_scheduler_type"] + == "linear" + ) + assert rl_client.tuning.training.resources == { + "charge_type": "mtu_postpaid", + "mtu_spec_code": "MTU4", + "mtu_capacity": 24, + } + + +def test_pg_opd_requires_teacher_model(rl_client): + rl_client.tuning.training.type = TrainingType.PG_OPD + + with pytest.raises( + ValueErrorWithCode, + match=f"teacher_model is required when training.type is {TrainingType.PG_OPD}", + ): + rl_client.submit_job(datasets=[_training_dataset()]) + + +def test_regular_reinforcement_is_unchanged(rl_client): + with patch.object( + CreateMixin, + "call", + return_value=_success_response(), + ) as call: + rl_client.submit_job(datasets=[_training_dataset()]) + + request = call.call_args.args[0] + assert request["training_type"] == "reinforcement" + assert "teacher_model" not in request + + +@pytest.mark.asyncio +async def test_run_forwards_teacher_model(rl_client): + submit = MagicMock(return_value="submitted") + + with ( + patch.object( + AgenticRL, + "register_functions", + new=AsyncMock(), + ), + patch.object( + AgenticRL, + "upload_datasets", + new=AsyncMock(), + ), + patch.object( + AgenticRL, + "submit_job", + new=submit, + ), + ): + result = await rl_client.run(teacher_model="qwen3.5-397b-a17b") + + assert result == "submitted" + assert submit.call_args.kwargs["teacher_model"] == "qwen3.5-397b-a17b"