8000 Fix target-length qualification metadata by DaoyuanLi2816 · Pull Request #112 · DaoyuanLi2816/mini-verl · GitHub
[go: up one dir, main page]

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 26 additions & 5 deletions src/miniverl/data/verl_parquet.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,11 @@

SplitName = Literal["train", "val"]
_PRESERVED_FIELDS = ("data_source", "ability", "reward_model", "extra_info")
_REWARD_MODEL_FIELDS = frozenset({"style", "ground_truth"})
_REWARD_MODEL_FIELDS_BY_STYLE = {
"exact": frozenset({"style", "ground_truth"}),
"rule": frozenset({"style", "ground_truth"}),
"target_length": frozenset({"style", "characters"}),
}
_REWARD_SCALARS = (str, int, float, bool)


Expand Down Expand Up @@ -48,17 +52,34 @@ def _validate_task_reward_metadata(preserved: dict[str, Any], *, location: str)
data_source = preserved["data_source"]
if not isinstance(data_source, str) or not data_source:
raise ConfigError(f"{location} requires a non-empty string data_source for task rewards")
unknown = sorted(set(reward_model).difference(_REWARD_MODEL_FIELDS))
style = reward_model.get("style")
if not isinstance(style, str):
raise ConfigError(
f"{location} reward_model.style is unsupported: {style!r}",
hint="use a deterministic built-in style: exact, rule, or target_length",
)
allowed_fields = _REWARD_MODEL_FIELDS_BY_STYLE.get(style)
if allowed_fields is None:
raise ConfigError(
f"{location} reward_model.style is unsupported: {style!r}",
hint="use a deterministic built-in style: exact, rule, or target_length",
)
unknown = sorted(set(reward_model).difference(allowed_fields))
if unknown:
raise ConfigError(
f"{location} reward_model contains unsupported fields: {', '.join(unknown)}",
hint="only deterministic exact/rule metadata is accepted; Python modules and code are not",
hint="only fields defined by the selected built-in reward style are accepted",
)
if reward_model.get("style") not in {"exact", "rule"}:
raise ConfigError(f"{location} reward_model.style must be exact/rule")
extra = preserved["extra_info"]
if extra is not None and not isinstance(extra, dict):
raise ConfigError(f"{location} extra_info must be an object or null")
if style == "target_length":
target = reward_model.get("characters")
if not isinstance(target, int) or isinstance(target, bool) or not 1 <= target <= 1_000_000:
raise Conf 8000 igError(
f"{location} target_length characters must be an integer in [1, 1000000]"
)
return
primary = reward_model.get("ground_truth")
secondary = extra.get("ground_truth") if isinstance(extra, dict) else None
if primary is None and secondary is None:
Expand Down
8 changes: 6 additions & 2 deletions tests/unit/test_v012_rl_qualification.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,14 @@ def test_v012_gpu_workload_source_compiles_to_validated_grpo_recipe(tmp_path: Pa
from miniverl.algorithms.contract import UPSTREAM_VERL_COMMIT
from miniverl.bridge.rl_v09 import publish_imported_verl_rl_v09
from miniverl.config import RunConfig
from scripts.run_v012_rl_qualification import _source_payload
from miniverl.data.verl_parquet import VerlParquetDataset
from scripts.run_v012_rl_qualification import _source_payload, _write_dataset

dataset = tmp_path / "data.parquet"
_write_dataset(dataset)
source = tmp_path / "source.yaml"
source.write_text(
yaml.safe_dump(_source_payload(tmp_path / "data.parquet"), sort_keys=False),
yaml.safe_dump(_source_payload(dataset), sort_keys=False),
encoding="utf-8",
)
recipe = tmp_path / "native.yaml"
Expand All @@ -32,3 +35,4 @@ def test_v012_gpu_workload_source_compiles_to_validated_grpo_recipe(tmp_path: Pa
assert config.rollout.samples_per_prompt == 4
assert config.train.cycles == 2
assert config.train.gradient_accumulation_steps == 8
assert VerlParquetDataset(config.source).inspect().rows == {"train": 4, "val": 0}
22 changes: 22 additions & 0 deletions tests/unit/test_verl_parquet_source.py
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,28 @@ def test_task_reward_row_accepts_bound_exact_ground_truth(tmp_path) -> None:
assert next(VerlParquetDataset(config).iter_split("train")).reward_model == row["reward_model"]


def test_task_reward_row_accepts_builtin_target_length_metadata(tmp_path) -> None:
row = _row(0)
row["reward_model"] = {"style": "target_length", "characters": 80}
path = tmp_path / "target-length.parquet"
_write(path, [row])
config = VerlParquetSourceConfig(train_files=[str(path)], use_task_rewards=True)

assert next(VerlParquetDataset(config).iter_split("train")).reward_model == row["reward_model"]


@pytest.mark.parametrize("target", [True, 0, -1, 1_000_001, "80"])
def test_task_reward_row_rejects_invalid_target_length_metadata(tmp_path, target: object) -> None:
row = _row(0)
row["reward_model"] = {"style": "target_length", "characters": target}
path = tmp_path / "target-length.parquet"
_write(path, [row])
config = VerlParquetSourceConfig(train_files=[str(path)], use_task_rewards=True)

with pytest.raises(ConfigError, match="characters"):
list(VerlParquetDataset(config).iter_split("train"))


def test_chat_template_is_applied_once_and_records_provenance(tmp_path) -> None:
train = _write(tmp_path / "train.parquet", [_row(1)])
source = VerlParquetSourceConfig(train_files=[str(train)], max_prompt_length=200)
Expand Down
0