8000 Fix RL training objectives and PPO updates by GeminiLight · Pull Request #56 · GeminiLight/virne · GitHub
[go: up one dir, main page]

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
128 changes: 127 additions & 1 deletion tests/solver/learning/test_rl_boundaries.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
NodeSlotsStepInstanceRLEnv,
)
from virne.solver.learning.rl_core.rl_enviroment_base import RLBaseEnv
from virne.solver.learning.rl_core.rl_solver import RLSolver
from virne.solver.learning.rl_core.rl_solver import A2CSolver, PPOSolver, RLSolver


NODE_ATTRS = [
Expand Down Expand Up @@ -233,6 +233,132 @@ def test_terminal_gae_characterization_values_are_stable():
np.testing.assert_allclose(buffer.returns, [2.62, 2.0])


def test_a2c_entropy_is_a_bonus_when_minimizing_the_loss():
solver = A2CSolver.__new__(A2CSolver)
solver.buffer = RolloutBuffer()
solver.buffer.observations = [{}, {}]
solver.buffer.actions = [0, 1]
solver.buffer.returns = [1.0, 2.0]
solver.preprocess_obs = lambda observations, device: observations
solver.device = torch.device('cpu')
solver.evaluate_actions = Mock(return_value=(
torch.tensor([1.0, 2.0]),
torch.tensor([0.0, 0.0]),
torch.tensor([0.4, 0.6]),
{},
))
solver.coef_critic_loss = 0.5
solver.coef_entropy_loss = 0.01
solver.config = OmegaConf.create({
'rl': {'norm_advantage': False},
'training': {'distributed_training': False},
})
solver.update_grad = Mock(return_value=torch.tensor(0.0))
solver.optimizer = SimpleNamespace(defaults={'lr': 0.001})
solver.logger = Mock()
solver.lr_scheduler = None
solver.update_time = 0

solver.update()

minimized_loss = solver.update_grad.call_args.args[0]
assert minimized_loss.item() == pytest.approx(-0.005)


def make_minimal_ppo_solver(current_values):
class PolicyThatMustNotBeCopied:
def __deepcopy__(self, memo):
raise AssertionError('PPO update must not copy an unused old policy')

solver = PPOSolver.__new__(PPOSolver)
solver.buffer = RolloutBuffer()
for index, (rollout_value, return_value) in enumerate(
zip([1.0, 2.0], [3.0, 5.0], strict=True)
):
solver.buffer.add(
{'index': index},
0,
0.0,
True,
np.array([0.0], dtype=np.float32),
value=rollout_value,
)
solver.buffer.returns.append(return_value)
solver.preprocess_obs = lambda observations, device: observations
solver.device = torch.device('cpu')
solver.policy = PolicyThatMustNotBeCopied()
solver.evaluate_actions = Mock(return_value=(
torch.tensor(current_values, dtype=torch.float32),
torch.tensor([0.2, -0.1]),
torch.zeros(2),
{},
))
solver.batch_size = 2
solver.repeat_times = 0
solver.eps_clip = 0.2
solver.coef_critic_loss = 0.0
solver.coef_entropy_loss = 0.0
solver.coef_mask_loss = 0.0
solver.criterion_critic = torch.nn.MSELoss()
solver.config = OmegaConf.create({
'rl': {
'norm_reward': False,
'norm_advantage': False,
'target_kl': None,
},
'training': {
'distributed_training': False,
'log_interval': 1,
},
})
solver.update_grad = Mock(return_value=torch.tensor(0.0))
solver.logger = Mock()
solver.lr_scheduler = None
solver.update_time = 0
return solver


def test_ppo_actor_advantages_do_not_drift_with_current_critic(monkeypatch):
monkeypatch.setattr(
torch,
'randint',
lambda *args, **kwargs: torch.tensor([0, 1]),
)
actor_losses = []
for current_values in ([10.0, 20.0], [-10.0, -20.0]):
solver = make_minimal_ppo_solver(current_values)

solver.update()

actor_losses.append(solver.logger.log.call_args.kwargs['data']['loss/actor_loss'])

np.testing.assert_allclose(actor_losses[0], actor_losses[1])


@pytest.mark.parametrize(
('buffer_size', 'batch_size', 'repeat_times', 'expected_updates'),
[
(128, 128, 10, 10),
(256, 128, 10, 20),
(129, 128, 10, 11),
(32, 128, 0, 1),
],
)
def test_ppo_update_count_uses_ceiling_without_an_extra_divisible_batch(
buffer_size,
batch_size,
repeat_times,
expected_updates,
):
solver = PPOSolver.__new__(PPOSolver)
solver.buffer = Mock()
solver.buffer.size.return_value = buffer_size
solver.batch_size = batch_size
solver.repeat_times = repeat_times

assert solver.calculate_update_sample_times() == expected_updates


def test_sync_vector_environment_uses_gymnasium_reset_and_step_contracts():
def make_env():
env = make_minimal_joint_env(candidate_nodes=[])
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,10 @@ def update(self, avg_cost):
batch_feasibiliy_flags = torch.FloatTensor(self.buffer.feasibility_flags)
batch_feasibility_budgets = torch.FloatTensor(self.buffer.feasibility_budgets)
batch_cost_violations = batch_cost_returns - self.cost_budget
batch_advantages = self.calculate_fixed_advantages(
batch_returns,
self.buffer.values,
)
# update the policy params repeatly
# sample_times = 1 + int(self.buffer.size() * self.repeat_times / self.batch_size)
sample_times = self.repeat_times
Expand All @@ -87,15 +91,16 @@ def update(self, avg_cost):
cost_returns = batch_cost_returns[sample_indices].to(self.device)
feasibility_flags = batch_feasibiliy_flags[sample_indices].to(self.device)
feasibility_budgets = batch_feasibility_budgets[sample_indices].to(self.device)
advantages = self.normalize_advantages(
batch_advantages[sample_indices].to(self.device)
)
old_action_logprobs = batch_old_action_logprobs[sample_indices].to(self.device)
# evaluate actions and observations
values, action_logprobs, dist_entropy, other = self.evaluate_actions(observations, actions, return_others=True)
cost_values = self.estimate_cost_with_grad(observations)
# calculate advantage
advantages = returns - values.detach()
cost_advantages = (cost_returns - self.cost_budget) - (cost_values.detach() - self.cost_budget)
if self.config.rl.norm_advantage and values.numel() != 0:
advantages = (advantages - advantages.mean()) / (advantages.std() + 1e-9)
if self.config.rl.norm_advantage and cost_advantages.numel() > 1:
cost_advantages = (cost_advantages - cost_advantages.mean()) / (cost_advantages.std() + 1e-9)
ratio = torch.exp(action_logprobs - old_action_logprobs)
surr1 = ratio * advantages
Expand Down Expand Up @@ -255,23 +260,25 @@ def update(self, ):
batch_returns = torch.FloatTensor(self.buffer.returns)
if self.config.rl.norm_reward:
batch_returns = (batch_returns - batch_returns.mean()) / (batch_returns.std() + 1e-9)
sample_times = 1 + int(self.buffer.size() * self.repeat_times / self.batch_size)
batch_advantages = self.calculate_fixed_advantages(
batch_returns,
self.buffer.values,
)
sample_times = self.calculate_update_sample_times()
for i in range(sample_times):
sample_indices = torch.randint(0, self.buffer.size(), size=(self.batch_size,)).long()
# observations = get_observations_sample(batch_observations, sample_indices, self.device)
sample_obersevations = [self.buffer.observations[i] for i in sample_indices]
observations = self.preprocess_obs(sample_obersevations, self.device)
actions = batch_actions[sample_indices].to(self.device)
returns = batch_returns[sample_indices].to(self.device)
advantages = self.normalize_advantages(
batch_advantages[sample_indices].to(self.device)
)
old_action_logprobs = batch_old_action_logprobs[sample_indices].to(self.device)
# evaluate actions and observations
values, action_logprobs, dist_entropy, other = self.evaluate_actions(observations, actions, return_others=True)

# calculate advantage
advantages = returns - values.detach()
if self.config.rl.norm_advantage and values.numel() != 0:
advantages = (advantages - advantages.mean()) / (advantages.std() + 1e-9)

ratio = torch.exp(action_logprobs - old_action_logprobs)
surr1 = ratio * advantages
surr2 = torch.clamp(ratio, 1. - self.eps_clip, 1. + self.eps_clip) * advantages
Expand Down Expand Up @@ -472,4 +479,4 @@ def obs_as_tensor(obs, device):
return {'hetero_data': obs_hetero_data, 'curr_v_node_id': obs_curr_v_node_id, 'action_mask': obs_action_mask, 'v_net_size': obs_v_net_size, 'aug_hetero_data_a': obs_aug_hetero_data_a, 'aug_hetero_data_b': obs_aug_hetero_data_b}
# return {'hetero_data': obs_hetero_data, 'curr_v_node_id': obs_curr_v_node_id, 'action_mask': obs_action_mask, 'v_net_size': obs_v_net_size}
else:
raise Exception(f"Unrecognized type of observation {type(obs)}")
raise Exception(f"Unrecognized type of observation {type(obs)}")
69 changes: 55 additions & 14 deletions virne/solver/learning/rl_core/rl_solver.py
F953
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import os
import csv
import copy
import math
import time
from collections.abc import Mapping
from omegaconf import OmegaConf, open_dict
Expand Down Expand Up @@ -224,6 +225,43 @@ def estimate_value(self, observation):
estimated_value = self.policy.evaluate(observation).squeeze(-1).detach().cpu().item()
return estimated_value

def calculate_fixed_advantages(self, returns, rollout_values):
"""Calculate policy advantages from values frozen during rollout collection."""
if len(rollout_values) != returns.numel():
raise ValueError(
'Expected one rollout value per return, but got '
f'{len(rollout_values)} values and {returns.numel()} returns.'
)
old_values = torch.tensor(
[
float(value.detach().cpu().reshape(-1)[0])
if isinstance(value, torch.Tensor)
else float(value)
for value in rollout_values
],
dtype=returns.dtype,
device=returns.device,
)
return returns - old_values

def normalize_advantages(self, advantages):
if self.config.rl.norm_advantage and advantages.numel() > 1:
return (
advantages - advantages.mean()
) / (advantages.std() + 1e-9)
return advantages

def calculate_update_sample_times(self):
"""Return the mini-batch updates needed for the configured PPO epochs."""
if self.buffer.size() <= 0:
raise ValueError('Cannot update a policy from an empty rollout buffer.')
if self.batch_size <= 0:
raise ValueError(f'batch_size must be positive, got {self.batch_size}.')
if self.repeat_times < 0:
raise ValueError(f'repeat_times must be non-negative, got {self.repeat_times}.')
target_samples = self.buffer.size() * self.repeat_times
return max(1, math.ceil(target_samples / self.batch_size))

def to_sub_solver(self):
temp_dict = {}
unnecessary_attributes = ['policy', 'optimizer', 'lr_scheduler', 'searcher', 'writer', 'logger']
Expand Down Expand Up @@ -480,7 +518,7 @@ def update(self, ):
actor_loss = - (action_logprobs * advantages).mean()
critic_loss = F.mse_loss(returns, values)
entropy_loss = dist_entropy.mean()
loss = actor_loss + self.coef_critic_loss * critic_loss + self.coef_entropy_loss * entropy_loss
loss = actor_loss + self.coef_critic_loss * critic_loss - self.coef_entropy_loss * entropy_loss

grad_clipped = self.update_grad(loss)

Expand Down Expand Up @@ -525,8 +563,6 @@ def calculate_kl_divergence(self, old_observations, old_policy):
def update(self, ):
# assert self.buffer.size() >= self.batch_size
device = torch.device('cpu')
# copy the old policy parameters
old_policy = copy.deepcopy(self.policy)

batch_observations = self.preprocess_obs(self.buffer.observations, device)
# # batch_actions = torch.LongTensor(np.concatenate(self.buffer.actions, axis=0)).to(self.device)
Expand All @@ -536,23 +572,25 @@ def update(self, ):
batch_returns = torch.FloatTensor(self.buffer.returns)
if self.config.rl.norm_reward:
batch_returns = (batch_returns - batch_returns.mean()) / (batch_returns.std() + 1e-9)
sample_times = 1 + int(self.buffer.size() * self.repeat_times / self.batch_size)
batch_advantages = self.calculate_fixed_advantages(
batch_returns,
self.buffer.values,
)
sample_times = self.calculate_update_sample_times()
for i in range(sample_times):
sample_indices = torch.randint(0, self.buffer.size(), size=(self.batch_size,)).long()
# observations = get_observations_sample(batch_observations, sample_indices, self.device)
sample_obersevations = [self.buffer.observations[i] for i in sample_indices]
observations = self.preprocess_obs(sample_obersevations, self.device)
actions = batch_actions[sample_indices].to(self.device)
returns = batch_returns[sample_indices].to(self.device)
advantages = self.normalize_advantages(
batch_advantages[sample_indices].to(self.device)
)
old_action_logprobs = batch_old_action_logprobs[sample_indices].to(self.device)
# evaluate actions and observations
values, action_logprobs, dist_entropy, other = self.evaluate_actions(observations, actions, return_others=True)

# calculate advantage
advantages = returns - values.detach()
if self.config.rl.norm_advantage and values.numel() != 0:
advantages = (advantages - advantages.mean()) / (advantages.std() + 1e-9)

ratio = torch.exp(action_logprobs - old_action_logprobs)
surr1 = ratio * advantages
surr2 = torch.clamp(ratio, 1. - self.eps_clip, 1. + self.eps_clip) * advantages
Expand Down Expand Up @@ -625,21 +663,24 @@ def update(self, ):
mean_batch_rewards = batch_rewards.mean()

batch_returns = torch.FloatTensor(self.buffer.returns)
sample_times = 1 + int(self.buffer.size() * self.repeat_times / self.batch_size)
batch_advantages = self.calculate_fixed_advantages(
batch_returns,
self.buffer.values,
)
sample_times = self.calculate_update_sample_times()
for i in range(sample_times):
sample_indices = torch.randint(0, self.buffer.size(), size=(self.batch_size,)).long()
observations = get_observations_sample(batch_observations, sample_indices, device=self.device)
actions = batch_actions[sample_indices].to(self.device)
returns = batch_returns[sample_indices].to(self.device)
advantages = self.normalize_advantages(
batch_advantages[sample_indices].to(self.device)
)
old_action_logprobs = batch_old_action_logprobs[sample_indices].to(self.device)
# masks = batch_masks[sample_indices].to(self.device) if batch_masks is not None else None
# evaluate actions and observations
values, action_logprobs, dist_entropy, other = self.evaluate_actions(observations, actions, return_others=True)

# calculate advantage
advantages = returns - values.detach()
if self.config.rl.norm_advantage and values.numel() != 0:
advantages = (advantages - advantages.mean()) / (advantages.std() + 1e-9)
ratio = torch.exp(action_logprobs - old_action_logprobs)
surr1 = ratio * advantages
surr2 = torch.clamp(ratio, 1. - self.eps_clip, 1. + self.eps_clip) * advantages
Expand Down
Loading
Loading
0