8000 mini-verl/src/miniverl/models/hf.py at main · gitlick/mini-verl · GitHub
[go: up one dir, main page]

Skip to content

Latest commit

 

History

History
635 lines (579 loc) · 25.7 KB

hf.py

File metadata and controls

635 lines (579 loc) · 25.7 KB
backbone = self.adapter.backbone
kwargs: dict[str, Any] = {"input_ids": input_ids, "use_cache": use_cache}
if attention_mask is not None:
kwargs["attention_mask"] = attention_mask
if past_key_values is not None:
kwargs["past_key_values"] = past_key_values
outputs = backbone(**kwargs)
hidden = getattr(outputs, "last_hidden_state", None)
if hidden is None:
hidden = outputs[0]
return hidden, getattr(outputs, "past_key_values", None)
# -- generation ---------------------------------------------------------
def generate(
self,
prefix_token_ids: Sequence[int],
*,
max_new_tokens: int,
stop_sequences: Sequence[str] = (),
temperature: float = 1.0,
top_p: float = 1.0,
top_k: int = 0,
seed: int | None = None,
record_logprobs: bool = False,
) -> GenerationOutput:
"""Sample a continuation, projecting one position per step."""
was_training = self.model.training
self.model.eval()
generator = torch.Generator(device="cpu")
generator.manual_seed(seed if seed is not None else 0)
cache_flag = getattr(self.model.config, "use_cache", True)
self.model.config.use_cache = True
def step(new_ids: list[int], state: Any) -> tuple[torch.Tensor, Any]:
ids = torch.tensor([new_ids], dtype=torch.long, device=self._device)
with torch.no_grad():
hidden, present = self._backbone_forward(ids, past_key_values=state, use_cache=True)
logits = self.adapter.lm_head(hidden[:, -1, :])
return logits[0].float(), present
try:
return run_generation(
step=step,
prefix_token_ids=prefix_token_ids,
decode=self.tokenizer.decode,
eos_token_id=self.tokenizer.eos_token_id,
max_new_tokens=max_new_tokens,
stop_sequences=stop_sequences,
temperature=temperature,
top_p=top_p,
top_k=top_k,
generator=generator,
record_logprobs=record_logprobs,
)
finally:
self.model.config.use_cache = cache_flag
if was_training:
self.model.train()
def generate_batch(
self,
prefix_token_ids: Sequence[Sequence[int]],
*,
max_new_tokens: int,
stop_sequences: Sequence[str] = (),
temperature: float = 0.0,
top_p: float = 1.0,
top_k: int = 0,
seeds: Sequence[int | None] | None = None,
record_logprobs: bool = False,
) -> list[GenerationOutput]:
"""Use one masked padded forward per greedy decoding step."""
if temperature != 0.0:
return super().generate_batch(
prefix_token_ids,
max_new_tokens=max_new_tokens,
stop_sequences=stop_sequences,
temperature=temperature,
top_p=top_p,
top_k=top_k,
seeds=seeds,
record_logprobs=record_logprobs,
)
was_training = self.model.training
self.model.eval()
def step(rows: list[list[int]]) -> torch.Tensor:
lengths = [len(row) for row in rows]
width = max(lengths)
ids = torch.full(
(len(rows), width),
self.tokenizer.pad_token_id,
dtype=torch.long,
device=self._device,
)
mask = torch.zeros((len(rows), width), dtype=torch.bool, device=self._device)
for index, row in enumerate(rows):
ids[index, : len(row)] = torch.tensor(row, dtype=torch.long, device=self._device)
mask[index, : len(row)] = True
with torch.no_grad():
hidden, _ = self._backbone_forward(ids, attention_mask=mask, use_cache=False)
positions = torch.tensor(
[length - 1 for length in lengths], dtype=torch.long, device=self._device
)
selected = hidden[torch.arange(len(rows), device=self._device), positions]
return self.adapter.lm_head(selected)
try:
return run_greedy_padded_generation(
step=step,
prefix_token_ids=prefix_token_ids,
decode=self.tokenizer.decode,
eos_token_id=self.tokenizer.eos_token_id,
max_new_tokens=max_new_tokens,
stop_sequences=stop_sequences,
record_logprobs=record_logprobs,
)
finally:
if was_training:
self.model.train()
# -- scoring ------------------------------------------------------------
def hidden_states_at(
self,
token_ids: Sequence[int],
positions: Sequence[int],
*,
with_grad: bool = False,
) -> torch.Tensor:
"""Backbone hidden states gathered at ``positions``; no full-sequence logits."""
ids = torch.tensor([list(token_ids)], dtype=torch.long, device=self._device)
index = torch.tensor(list(positions), dtype=torch.long, device=self._device)
context = torch.enable_grad() if with_grad else torch.no_grad()
with context:
hidden, _ = self._backbone_forward(ids, use_cache=False)
return hidden[0].index_select(0, index)
def hidden_states_at_batch(
self,
batch: Any,
*,
with_grad: bool = False,
) -> torch.Tensor:
"""Gather selected states from one masked padded backbone forward."""
context = torch.enable_grad() if with_grad else torch.no_grad()
with context:
hidden, _ = self._backbone_forward(
batch.input_ids,
attention_mask=batch.attention_mask,
use_cache=False,
)
return hidden[batch.selected_batch_indices, batch.selected_positions]
def project(self, hidden: torch.Tensor) -> torch.Tensor:
"""Apply the LM head to selected hidden states."""
head = self.adapter.lm_head
weight_dtype = getattr(head, "weight", None)
if weight_dtype is not None and hidden.dtype != weight_dtype.dtype:
hidden = hidden.to(weight_dtype.dtype)
return head(hidden)
# -- training -----------------------------------------------------------
def set_train(self, mode: bool) -> None:
"""Switch train/eval mode."""
self.model.train(mode)
def trainable_parameters(self) -> list[torch.nn.Parameter]:
"""Parameters requiring gradients (the LoRA adapters under QLoRA)."""
return [p for p in self.model.parameters() if p.requires_grad]
def trainable_state_dict(self) -> dict[str, torch.Tensor]:
"""CPU copy of the trainable weights only."""
return {
name: param.detach().to("cpu").clone()
for name, param in self.model.named_parameters()
if param.requires_grad
}
def load_trainable_state_dict(self, state: dict[str, torch.Tensor]) -> None:
"""Restore trainable weights in place."""
own = dict(self.model.named_parameters())
expected = {name for name, parameter in own.items() if parameter.requires_grad}
unknown = sorted(set(state).difference(expected))
missing = sorted(expected.difference(state))
if unknown:
raise BackendError(
f"checkpoint contains {len(unknown)} unknown parameter names, e.g. {unknown[0]}",
hint="the checkpoint was written by a different model or LoRA config",
)
if missing:
raise BackendError(
f"checkpoint is missing {len(missing)} trainable parameters, e.g. {missing[0]}",
hint="the checkpoint is incomplete or was written by a different LoRA config",
)
mismatched = [
(name, tuple(value.shape), tuple(own[name].shape))
for name, value in state.items()
if value.shape != own[name].shape
]
if mismatched:
name, actual, expected_shape = mismatched[0]
raise BackendError(
f"checkpoint parameter {name!r} has shape {actual}, expected {expected_shape}",
hint="the checkpoint was written by a different model or LoRA config",
)
with torch.no_grad():
for name, value in state.items():
own[name].copy_(value.to(own[name].device, own[name].dtype))
# -- placement -----------------------------------------------------------
def to_device(self, device: str) -> None:
"""Move the model, unless it is bitsandbytes-quantized (pinned at load)."""
if self.capabilities.quantization != "none" and device != self._device:
raise BackendError(
"a bitsandbytes-quantized model cannot be moved between devices after loading",
hint="use memory.strategy: resident for quantized models, or load the "
"model unquantized",
)
self.model.to(device)
self._device = device
self.capabilities.device = device
def release(self) -> None:
"""Move to host memory and free the CUDA caching allocator's blocks."""
from miniverl.utils.gpu import empty_cache
if self.capabilities.quantization == "none":
self.model.to("cpu")
self._device = "cpu"
self.capabilities.device = "cpu"
empty_cache()
@property
def device(self) -> str:
"""Current device."""
return self._device
0