Why critic is updated each timestep?

[xihuai18]

pymarl/src/learners/coma_learner.py

Lines 118 to 148 in e633fdf

	for t in reversed(range(rewards.size(1))):
	mask_t = mask[:, t].expand(-1, self.n_agents)
	if mask_t.sum() == 0:
	continue
	q_t = self.critic(batch, t)
	q_vals[:, t] = q_t.view(bs, self.n_agents, self.n_actions)
	q_taken = th.gather(q_t, dim=3, index=actions[:, t:t+1]).squeeze(3).squeeze(1)
	targets_t = targets[:, t]
	td_error = (q_taken - targets_t.detach())
	# 0-out the targets that came from padded data
	masked_td_error = td_error * mask_t
	# Normal L2 loss, take mean over actual data
	loss = (masked_td_error ** 2).sum() / mask_t.sum()
	self.critic_optimiser.zero_grad()
	loss.backward()
	grad_norm = th.nn.utils.clip_grad_norm_(self.critic_params, self.args.grad_norm_clip)
	self.critic_optimiser.step()
	self.critic_training_steps += 1
	running_log["critic_loss"].append(loss.item())
	running_log["critic_grad_norm"].append(grad_norm)
	mask_elems = mask_t.sum().item()
	running_log["td_error_abs"].append((masked_td_error.abs().sum().item() / mask_elems))
	running_log["q_taken_mean"].append((q_taken * mask_t).sum().item() / mask_elems)
	running_log["target_mean"].append((targets_t * mask_t).sum().item() / mask_elems)
	return q_vals, running_log