8000 Complete discrete DQN and DDPG solvers by GeminiLight · Pull Request #57 · GeminiLight/virne · GitHub
[go: up one dir, main page]

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions docs/source/intro/rl-support.rst
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,21 @@ requests, saves a checkpoint, and then evaluates the trained policy:
require larger training and evaluation sets, controlled seeds, and the
protocol described in the benchmark paper.

Discrete Off-policy Solvers
---------------------------

Virne also registers ``dqn_mlp+``, ``double_dqn_mlp+``, and ``ddpg_mlp+`` for
the physical-node action space. They use persistent replay memory and target
networks; invalid physical-node actions are masked during both exploration and
Bellman target calculation. Their settings live under ``rl.dqn`` and
``rl.ddpg`` in ``virne/configs/learning.yaml``.

``ddpg_mlp+`` is a discrete-action adaptation, not vanilla continuous-action
DDPG. Its actor emits masked node logits, its critic estimates one Q-value per
physical node, and the actor is optimized through a differentiable categorical
relaxation. This distinction should be retained when reporting the algorithm
in experiments.

The final checkpoint is written to:

.. code-block:: text
Expand Down
218 changes: 218 additions & 0 deletions tests/solver/learning/test_off_policy_solvers.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,218 @@
from copy import deepcopy
from types import SimpleNamespace

import numpy as np
import pytest
import torch
from omegaconf import OmegaConf

from virne.solver.learning.rl_core.buffer import RolloutBuffer
from virne.solver.learning.rl_core.instance_agent import InstanceAgent
from virne.solver.learning.rl_core.rl_solver import DDPGSolver, DQNSolver


class TinyActionPolicy(torch.nn.Module):
def __init__(self) -> None:
super().__init__()
self.actor = torch.nn.Linear(1, 1)

def act(self, observation):
return self.actor(observation['features']).squeeze(-1)


def make_off_policy_solver(solver_cls, algorithm_name):
solver = solver_cls.__new__(solver_cls)
solver.algorithm_name = algorithm_name
solver.config = OmegaConf.create(
{
'training': {
'distributed_training': False,
'log_interval': 1,
},
'rl': {
'gamma': 0.9,
'mask_actions': True,
'clip_grad': True,
'max_grad_norm': 1.0,
'weight_decay': 0.0,
'learning_rate': {'actor': 0.01, 'critic': 0.01},
},
},
)
solver.device = torch.device('cpu')
solver.batch_size = 2
solver.buffer = RolloutBuffer()
solver.policy = TinyActionPolicy()
solver.optimizer = torch.optim.Adam(solver.policy.parameters(), lr=0.01)
solver.preprocess_obs = lambda observations, device: {
'features': torch.as_tensor(
np.stack([observation['features'] for observation in observations]),
dtype=torch.float32,
device=device,
),
'action_mask': torch.as_tensor(
np.stack([observation['action_mask'] for observation in observations]),
dtype=torch.bool,
device=device,
),
}
solver.logger = SimpleNamespace(log=lambda *args, **kwargs: None)
solver.update_time = 0
solver.epsilon = 0.0
return solver


def add_transitions(solver):
transitions = [
(
{'features': np.array([[1.0], [2.0]]), 'action_mask': np.array([True, True])},
0,
1.0,
False,
{'features': np.array([[2.0], [1.0]]), 'action_mask': np.array([False, True])},
),
(
{'features': np.array([[2.0], [1.0]]), 'action_mask': np.array([True, True])},
1,
-0.5,
True,
{'features': np.array([[3.0], [4.0]]), 'action_mask': np.array([True, True])},
),
]
for observation, action, reward, done, next_observation in transitions:
solver.buffer.add(
observation,
action,
reward,
done,
np.zeros((1, 1), dtype=np.float32),
value=0.0,
next_obs=next_observation,
)


def test_rollout_buffer_trim_preserves_newest_aligned_transitions():
buffer = RolloutBuffer()
for index in range(4):
buffer.add(index, index, index, False, index, index, index + 1)

buffer.trim(2)

assert buffer.size() == 2
assert buffer.curr_idx == 2
assert buffer.observations == [2, 3]
assert buffer.actions == [2, 3]
assert buffer.next_observations == [3, 4]


def test_off_policy_merge_keeps_failed_experience_without_gae():
agent = InstanceAgent.__new__(InstanceAgent)
agent.is_off_policy = True
agent.replay_capacity = 2
agent.buffer = RolloutBuffer()
agent.time_step = 0
instance_buffer = RolloutBuffer()
for index in range(3):
instance_buffer.add(index, index, -1.0, True, 0.0, next_obs=index + 1)

agent.merge_instance_experience(
instance=None,
solution={'result': False},
instance_buffer=instance_buffer,
last_value=0.0,
)

assert agent.buffer.observations == [1, 2]
assert agent.buffer.returns == []
assert agent.time_step == 1


def test_dqn_exploration_samples_only_legal_actions():
solver = make_off_policy_solver(DQNSolver, 'dqn')
solver.epsilon_start = 1.0
solver.epsilon_end = 1.0
solver.epsilon_decay = 1.0
solver.steps_done = 0
observation = {
'features': torch.tensor([[[1.0], [2.0], [3.0]]]),
'action_mask': torch.tensor([[False, True, False]]),
}

actions = [solver.select_action(observation, sample=True)[0] for _ in range(20)]

assert actions == [1] * 20


def test_dqn_update_uses_replay_without_clearing_it():
solver = make_off_policy_solver(DQNSolver, 'dqn')
solver.target_policy = deepcopy(solver.policy)
solver.target_update_interval = 1
add_transitions(solver)
old_parameters = [parameter.detach().clone() for parameter in solver.policy.parameters()]

loss = solver.update()

assert torch.isfinite(loss)
assert solver.buffer.size() == 2
assert any(
not torch.equal(before, after)
for before, after in zip(old_parameters, solver.policy.parameters(), strict=True)
)
assert all(
torch.equal(online, target)
for online, target in zip(
solver.policy.state_dict().values(),
solver.target_policy.state_dict().values(),
strict=True,
)
)


def test_dqn_checkpoint_metadata_is_safe_for_weights_only_loading(tmp_path):
solver = make_off_policy_solver(DQNSolver, 'dqn')
solver.target_policy = deepcopy(solver.policy)
solver.steps_done = 3
solver.epsilon = np.float64(0.25)
checkpoint_path = tmp_path / 'dqn.pkl'

torch.save(solver.get_additional_checkpoint_state(), checkpoint_path)

checkpoint = torch.load(checkpoint_path, weights_only=True)
assert checkpoint['steps_done'] == 3
assert checkpoint['epsilon'] == 0.25


def test_discrete_ddpg_updates_actor_and_critic_and_soft_targets():
solver = make_off_policy_solver(DDPGSolver, 'ddpg')
solver.critic_policy = TinyActionPolicy()
solver.critic_optimizer = torch.optim.Adam(
solver.critic_policy.parameters(),
lr=0.01,
)
solver.target_policy = deepcopy(solver.policy)
solver.target_critic_policy = deepcopy(solver.critic_policy)
solver.tau = 0.2
solver.policy_temperature = 1.0
add_transitions(solver)
actor_before = [parameter.detach().clone() for parameter in solver.policy.parameters()]
critic_before = [parameter.detach().clone() for parameter in solver.critic_policy.parameters()]

loss = solver.update()

assert torch.isfinite(loss)
assert solver.buffer.size() == 2
assert any(
not torch.equal(before, after)
for before, after in zip(actor_before, solver.policy.parameters(), strict=True)
)
assert any(
not torch.equal(before, after)
for before, after in zip(critic_before, solver.critic_policy.parameters(), strict=True)
)


@pytest.mark.parametrize('solver_name', ['dqn_mlp+', 'ddpg_mlp+'])
def test_off_policy_mlp_solvers_are_registered(solver_name):
from virne.solver import SolverRegistry

assert SolverRegistry.get(solver_name).type == 'r_learning'
20 changes: 18 additions & 2 deletions virne/configs/learning.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -71,8 +71,24 @@ rl:
target_kl: null
gae_lambda: 0.98
repeat_times: 10
# DQN
explore_rate: 0.9
# Off-policy algorithms for the discrete physical-node action space
dqn:
replay_capacity: 10000
learning_starts: 128
target_update_interval: 100
epsilon_start: 1.0
epsilon_end: 0.1
epsilon_decay: 10000
gradient_steps_per_transition: 1
ddpg:
replay_capacity: 10000
learning_starts: 128
tau: 0.005
epsilon_start: 1.0
epsilon_end: 0.1
epsilon_decay: 10000
policy_temperature: 1.0
gradient_steps_per_transition: 1
# Safe RL
safe_rl:
srl_cost_budget: 0.2
Expand Down
21 changes: 17 additions & 4 deletions virne/solver/learning/reinforcement_learning/mlp_solver.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,15 @@
from virne.solver import SolverRegistry
from virne.solver.learning.rl_policy import MlpActorCritic
from virne.solver.learning.rl_core import JointPRStepInstanceRLEnv, PlaceStepInstanceRLEnv
from virne.solver.learning.rl_core.rl_solver import PGSolver, A2CSolver, PPOSolver, A3CSolver
from virne.solver.learning.rl_core.rl_solver import (
A2CSolver,
A3CSolver,
DDPGSolver,
DQNSolver,
DoubleDQNSolver,
PGSolver,
PPOSolver,
)
from virne.solver.learning.rl_core.instance_agent import InstanceAgent
from virne.solver.learning.rl_core.tensor_convertor import TensorConvertor
from virne.solver.learning.rl_core.policy_builder import PolicyBuilder
Expand Down Expand Up @@ -94,14 +102,19 @@ def __init__(self, p_net: PhysicalNetwork, v_net: VirtualNetwork, controller: Co
{'solver_name': 'pg_mlp+', 'policy_key': 'mlp', 'solver_cls_name': 'PgMlpSolver', 'rl_solver_cls': PGSolver},
{'solver_name': 'ppo_mlp+', 'policy_key': 'mlp', 'solver_cls_name': 'PpoMlpSolver', 'rl_solver_cls': PPOSolver},
{'solver_name': 'a2c_mlp+', 'policy_key': 'mlp', 'solver_cls_name': 'A2cMlpSolver', 'rl_solver_cls': A2CSolver},
{'solver_name': 'a3c_mlp+', 'policy_key': 'mlp', 'solver_cls_name': 'A3cMlpSolver', 'rl_solver_cls': A3CSolver}
{'solver_name': 'a3c_mlp+', 'policy_key': 'mlp', 'solver_cls_name': 'A3cMlpSolver', 'rl_solver_cls': A3CSolver},
{'solver_name': 'dqn_mlp+', 'policy_key': 'mlp', 'solver_cls_name': 'DqnMlpSolver', 'rl_solver_cls': DQNSolver},
{'solver_name': 'double_dqn_mlp+', 'policy_key': 'mlp', 'solver_cls_name': 'DoubleDqnMlpSolver', 'rl_solver_cls': DoubleDQNSolver},
{'solver_name': 'ddpg_mlp+', 'policy_key': 'mlp', 'solver_cls_name': 'DdpgMlpSolver', 'rl_solver_cls': DDPGSolver},
]
for solver_info in extended_mlp_solvers:
# make and register the solver class
solver_name = solver_info['solver_name']
policy_key = '_'.join(solver_info['solver_name'].split('_')[1:])[:-1]
policy_builder = build_policy
base_solver_cls = solver_info['rl_solver_cls']
policy_builder = PolicyBuilder.build_mlp_action_policy \
if issubclass(base_solver_cls, (DQNSolver, DDPGSolver)) \
else build_policy
obs_as_tensor = TensorConvertor.obs_as_tensor_for_mlp
instance_env_cls = MlpInstanceEnv
base_solver_cls = solver_info['rl_solver_cls']
make_solver_class(solver_name, instance_env_cls, base_solver_cls, policy_builder, obs_as_tensor)
13 changes: 13 additions & 0 deletions virne/solver/learning/rl_core/buffer.py
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,19 @@ def merge(self, buffer):
main_item_list = getattr(self, item)
sub_item_list = getattr(buffer, item)
main_item_list += sub_item_list
self.curr_idx = self.size()

def trim(self, max_size):
"""Keep only the newest ``max_size`` transitions in the buffer."""
if max_size <= 0:
raise ValueError(f'max_size must be positive, got {max_size}.')
if self.size() <= max_size:
return
for item in self.all_items:
item_list = getattr(self, item)
if len(item_list) > max_size:
setattr(self, item, item_list[-max_size:])
self.curr_idx = self.size()

def compute_returns_and_advantages(self, last_value=0., gamma=0.99, gae_lambda=0.98, method='gae', values=None) -> None:
"""
Expand Down
16 changes: 14 additions & 2 deletions virne/solver/learning/rl_core/instance_agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,8 @@ def learn_with_instance(self, instance):
feature_time = time_b - time_a
# print(feature_time)

value = self.estimate_value(tensor_instance_obs)
value = 0.0 if getattr(self, 'is_off_policy', False) \
else self.estimate_value(tensor_instance_obs)
(
next_instance_obs,
instance_reward,
Expand Down Expand Up @@ -162,6 +163,14 @@ def learn_with_instance(self, instance):
return solution, instance_buffer, last_value

def merge_instance_experience(self, instance, solution, instance_buffer, last_value):
if getattr(self, 'is_off_policy', False):
# Off-policy algorithms learn from both successful and failed
# transitions and keep them across updates as replay memory.
self.buffer.merge(instance_buffer)
self.buffer.trim(self.replay_capacity)
self.time_step += 1
return self.buffer

### -- if_use_negative_sample -- ##
if self.config.rl.if_use_negative_sample:
baseline_solution_info = self.get_baseline_solution_info(instance, self.if_use_baseline_solver)
Expand Down Expand Up @@ -203,7 +212,10 @@ def learn_singly(self, env, num_epochs=1, **kwargs):
# update parameters
# env.v_net.num_nodes | env.v_net_simulator.v_nets[0].num_nodes
if self.buffer.size() >= self.target_steps:
loss = self.update()
update_steps = self.get_update_steps(instance_buffer) \
if hasattr(self, 'get_update_steps') else 1
for _ in range(update_steps):
loss = self.update()

instance, reward, done, info = env.step(solution)

Expand Down
15 changes: 14 additions & 1 deletion virne/solver/learning/rl_core/policy_builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import torch.nn as nn

from virne.solver.learning.rl_policy.gnn_mlp_policy import DeepEdgeFeatureGATActorCritic
from ..rl_policy import GcnMlpActorCritic, GatMlpActorCritic, MlpActorCritic, CnnActorCritic, AttActorCritic
from ..rl_policy import GcnMlpActorCritic, GatMlpActorCritic, MlpActionPolicy, MlpActorCritic, CnnActorCritic, AttActorCritic
from ..rl_policy import BiGcnActorCritic, BiGatActorCritic, BiDeepEdgeFeatureGatActorCritic
from ..utils import get_pyg_data
from ..obs_handler import POSITIONAL_EMBEDDING_DIM, P_NODE_STATUS_DIM, V_NODE_STATUS_DIM, V_NET_STATUS_DIM
Expand Down Expand Up @@ -89,6 +89,19 @@ def build_mlp_policy(agent: Any) -> Tuple[nn.Module, torch.optim.Optimizer]:
optimizer = OptimizerBuilder.build_optimizer(agent.config, policy)
return policy, optimizer

@staticmethod
def build_mlp_action_policy(agent: Any) -> Tuple[nn.Module, torch.optim.Optimizer]:
"""Build per-node action scores for DQN and discrete DDPG."""
p_net_x_dim = get_p_net_x_dim(agent.config)
v_node_feature_dim = get_v_node_x_dim(agent.config)
policy = MlpActionPolicy(
feature_dim=p_net_x_dim + v_node_feature_dim,
action_dim=agent.config.simulation.p_net_setting_num_nodes,
**PolicyBuilder.get_general_nn_config(agent.config),
).to(agent.device)
optimizer = OptimizerBuilder.build_optimizer(agent.config, policy)
return policy, optimizer

@staticmethod
def build_cnn_policy(agent: Any) -> Tuple[nn.Module, torch.optim.Optimizer]:
"""
Expand Down
Loading
Loading
0