feat: support PG-OPD unified training flow

dashscope/cli/agentic_rl.py

@@ -460,6 +460,14 @@ def run(
         "--config",
         help="Path to YAML configuration file",
     ),
+    teacher_model: Optional[str] = typer.Option(
+        None,
+        "--teacher-model",
+        help=(
+            "Enable OPD with this teacher model (overrides YAML); "
+            "Rollout and Reward are optional"
+        ),
+    ),
     job_name: Optional[str] = typer.Option(
         None,
         help="Custom name for the tuning job",
@@ -491,10 +499,8 @@ def run(
     1. Configuration-driven: Use -c/--config to specify a YAML file
     2. Direct parameter: Provide all required arguments via CLI options
 
-    Required parameters:
-    - rollout_classpath
-    - reward_classpaths (at least one)
-    - training_files (at least one)
+    Rollout and Reward are required for regular reinforcement learning, but
+    optional for OPD.
     """
     output_format = validate_output_format(output_format)
     _apply_verbose(verbose)
@@ -505,6 +511,7 @@ def run(
 
     # Prepare workflow parameters
     run_kwargs = {
+        "teacher_model": teacher_model,
         "job_name": job_name,
     }
 

dashscope/finetune/agentic_rl.py

@@ -28,6 +28,7 @@
 from dashscope.finetune.reinforcement import (
     FunctionType,
     DatasetsType,
+    TrainingType,
 )
 from dashscope.finetune.reinforcement import (
     RewardInput,
@@ -171,11 +172,14 @@ def submit_job(
         hyper_parameters: Optional[Dict[str, str]] = None,
         resources: Optional[Dict[str, str]] = None,
         job_name: Optional[str] = None,
+        teacher_model: Optional[str] = None,
         **kwargs,
     ) -> FineTune:
         """
         Submit RL tuning job to the platform.
         """
+        teacher_model = teacher_model or self.tuning.teacher_model
+
         # Resolve job name (fallback to class default)
         if job_name:
             self.tuning.name = job_name
@@ -230,6 +234,15 @@ def submit_job(
         if resources:
             self.tuning.training.resources = resources
 
+        training_type = self.tuning.training.type
+        if teacher_model:
+            training_type = TrainingType.PG_OPD
+        elif training_type == TrainingType.PG_OPD:
+            raise ValueErrorWithCode(
+                f"teacher_model is required when training.type is {training_type}",
+                error_code=3006,
+            )
+
         request = {
             "model": self.tuning.model.name,
             "training_datasets": [ds.model_dump() for ds in training_datasets],
@@ -240,7 +253,8 @@ def submit_job(
             "rewards": rewards,
             "hyper_parameters": self.tuning.training.hyper_parameters,
             "resource_config": self.tuning.training.resources,
-            "training_type": str(self.tuning.training.type),
+            "training_type": str(training_type),
+            "teacher_model": teacher_model,
             "job_name": job_name_with_suffix,
         }
         request = deep_remove_none(request)
@@ -287,6 +301,7 @@ async def run(
         hyper_parameters: Optional[Dict[str, str]] = None,
         resources: Optional[Dict[str, str]] = None,
         job_name: Optional[str] = None,
+        teacher_model: Optional[str] = None,
         **kwargs,
     ) -> FineTune:
         """
@@ -315,6 +330,7 @@ async def run(
                 hyper_parameters=hyper_parameters,
                 resources=resources,
                 job_name=job_name,
+                teacher_model=teacher_model,
                 **kwargs,
             )
         except Exception as e:

dashscope/finetune/reinforcement/common/model.py

@@ -1397,6 +1397,7 @@ class TuningModel(Models, BaseModel):
 
     name: str = Field(default="agentic-rl", min_length=1, max_length=256)
     model: FoundationModel = FoundationModel()
+    teacher_model: Optional[str] = None
     functions: List[FunctionComponentUnion] = Field(default_factory=list)
     datasets: List[Dataset] = Field(default_factory=list)
     training: Training = Training()

dashscope/finetune/reinforcement/common/model_types.py

@@ -47,6 +47,7 @@ def __str__(self):
 
 class TrainingType(str, Enum):
     TRAINING_TYPE = "reinforcement"
+    PG_OPD = "pg_opd"
 
     def __str__(self):
         return self.value

dashscope/finetune/reinforcement/common/utils.py

@@ -381,14 +381,15 @@ def create_deployment_files(
             function_layer_used=FC_LAYER_USED,
         )
 
-        with open(FC_FILES_START, "w", encoding="utf-8") as f:
+        start_file = os.path.join(dirpath, FC_FILES_START)
+        with open(start_file, "w", encoding="utf-8") as f:
             f.write(content)
 
         # Add execute permission on Unix systems
         if os.name == "posix":
-            os.chmod(FC_FILES_START, 0o755)
+            os.chmod(start_file, 0o755)
 
-        logger.debug(f"Generated startup script: {FC_FILES_START}")
+        logger.debug(f"Generated startup script: {start_file}")
     except Exception as e:
         raise RuntimeErrorWithCode(
             "Deployment file creation error",

dashscope/finetune/reinforcement/examples/workspace/README-zh.md

@@ -56,6 +56,16 @@ langchain-openai==1.2.0
 
 > 注：functions/目录下需要包含__init__.py文件
 
+函数组件是否必需取决于训练配置：
+
+| 配置 | 场景 | 自定义 Rollout | 自定义 Reward |
+|---|---|---:|---:|
+| `rl-job.yaml` | 普通强化学习 | 必选 | 至少一个 |
+| `opd-job.yaml` 不保留函数块 | 仅 Teacher | 否 | 否 |
+| `opd-job.yaml` 仅保留 Reward | Teacher + Reward | 否 | 是 |
+| `opd-job.yaml` 仅保留 Rollout | Teacher + Rollout | 是 | 否 |
+| `opd-job.yaml` 保留两个函数块 | Teacher + Rollout + Reward | 是 | 是 |
+
 ### 3.2 准备训练数据
 
 在`data`目录下添加数据集文件：
@@ -67,19 +77,56 @@ langchain-openai==1.2.0
 
 ### 4.1 函数执行（注册+测试）
 
+普通强化学习必须注册自定义 Rollout 和至少一个 Reward。OPD 的
+Rollout、Reward 可选。
+
 ```bash
 python test_functions.py
 ```
 
 ### 4.2 工作流执行（YAML配置+生命周期管理）
 
+普通强化学习与 OPD 使用完全相同的 SDK 工作流，只需选择对应 YAML：
+
+```python
+from dashscope.finetune.agentic_rl import AgenticRL
+
+client = AgenticRL()
+# 普通强化学习使用 rl-job.yaml；OPD 使用 opd-job.yaml
+client.init(config_path="rl-job.yaml")
+result = await client.run()
+```
+
+`opd-job.yaml` 默认保留两个函数块，表示 Teacher + Rollout + Reward。删除
+Reward 块表示 Teacher + Rollout；删除 Rollout 块表示 Teacher + Reward；
+两个都删除表示仅 Teacher。
+
 ```bash
+# submit_job.py 默认使用普通强化学习 rl-job.yaml
 python submit_job.py
 ```
 
 ## 5. 使用CLI执行任务
 示例代码：cli.sh
 
+`cli.sh` 展示普通强化学习的完整流程。执行 OPD 时，对 `opd-job.yaml` 中已经
+删除的函数块跳过对应注册和测试；数据提交、`rl run` 和任务生命周期命令
+保持不变。
+
+CLI 与 SDK 使用同一份 YAML：
+
+```bash
+# 普通强化学习
+dashscope rl run -c rl-job.yaml
+
+# OPD
+dashscope rl run -c opd-job.yaml
+
+# 覆盖 opd-job.yaml 中配置的 Teacher 模型
+dashscope rl run -c opd-job.yaml \
+  --teacher-model qwen3.5-397b-a17b
+```
+
 ```bash
 dashscope rl --help  # 查看完整命令帮助
 

dashscope/finetune/reinforcement/examples/workspace/README.md

@@ -56,6 +56,16 @@ Develop functions under the `functions` directory:
 
 > Note: The `functions/` directory must contain an `__init__.py` file
 
+Function components depend on the selected training configuration:
+
+| Configuration | Mode | Custom Rollout | Custom Reward |
+|---|---|---:|---:|
+| `rl-job.yaml` | Regular reinforcement | Required | At least one |
+| `opd-job.yaml` without function blocks | Teacher only | No | No |
+| `opd-job.yaml` with Reward only | Teacher + Reward | No | Yes |
+| `opd-job.yaml` with Rollout only | Teacher + Rollout | Yes | No |
+| `opd-job.yaml` with both blocks | Teacher + Rollout + Reward | Yes | Yes |
+
 ### 3.2 Preparing Training Data
 
 Add dataset files under the `data` directory:
@@ -67,19 +77,57 @@ Add dataset files under the `data` directory:
 
 ### 4.1 Function Execution (Register + Test)
 
+Regular reinforcement training requires a custom Rollout and at least one
+Reward. Rollout and Reward are optional for OPD.
+
 ```bash
 python test_functions.py
 ```
 
 ### 4.2 Workflow Execution (YAML Config + Lifecycle Management)
 
+Regular reinforcement training and OPD use the same SDK workflow. Select the
+corresponding YAML:
+
+```python
+from dashscope.finetune.agentic_rl import AgenticRL
+
+client = AgenticRL()
+# Use rl-job.yaml for regular reinforcement or opd-job.yaml for OPD
+client.init(config_path="rl-job.yaml")
+result = await client.run()
+```
+
+`opd-job.yaml` keeps both function blocks by default, representing Teacher +
+Rollout + Reward. Remove the Reward block for Teacher + Rollout, remove the
+Rollout block for Teacher + Reward, or remove both for Teacher only.
+
 ```bash
+# submit_job.py uses the regular reinforcement rl-job.yaml by default
 python submit_job.py
 ```
 
 ## 5. Executing Tasks with CLI
 Example code：cli.sh
 
+`cli.sh` demonstrates the regular reinforcement workflow. For OPD, skip
+registration and testing for function blocks removed from `opd-job.yaml`; dataset
+submission, `rl run`, and job lifecycle commands remain unchanged.
+
+The CLI consumes the same YAML files as the SDK:
+
+```bash
+# Regular reinforcement
+dashscope rl run -c rl-job.yaml
+
+# OPD
+dashscope rl run -c opd-job.yaml
+
+# Override the Teacher configured in opd-job.yaml
+dashscope rl run -c opd-job.yaml \
+  --teacher-model qwen3.5-397b-a17b
+```
+
 ```bash
 dashscope rl --help  # View full command help