Skip to content

Commit

Permalink
moved logs connected to unknown undeployable to Devel
Browse files Browse the repository at this point in the history
  • Loading branch information
Michal Tichák committed Oct 28, 2024
1 parent ddb067b commit f03e42c
Show file tree
Hide file tree
Showing 3 changed files with 15 additions and 9 deletions.
3 changes: 3 additions & 0 deletions core/environment/transition_deploy.go
Original file line number Diff line number Diff line change
Expand Up @@ -245,6 +245,7 @@ func (t DeployTransition) do(env *Environment) (err error) {
WithField("partition", env.Id().String()).
WithField("timeout", deploymentTimeout).
WithField("detector", detector).
WithField("level", infologger.IL_Devel).
Error("role failed to deploy within timeout")
undeployableTaskRoles = append(undeployableTaskRoles, role.GetPath())
}
Expand Down Expand Up @@ -275,6 +276,7 @@ func (t DeployTransition) do(env *Environment) (err error) {
WithField("partition", env.Id().String()).
WithField("timeout", deploymentTimeout).
WithField("detector", detector).
WithField("level", infologger.IL_Devel).
Error("role failed to deploy because of timeout")
undeployableTaskRoles = append(undeployableTaskRoles, role.GetPath())
} else if roleStatus != task.ACTIVE {
Expand All @@ -288,6 +290,7 @@ func (t DeployTransition) do(env *Environment) (err error) {
WithField("partition", env.Id().String()).
WithField("timeout", deploymentTimeout).
WithField("detector", detector).
WithField("level", infologger.IL_Devel).
Error("role failed to deploy because of timeout")
inactiveTaskRoles = append(inactiveTaskRoles, role.GetPath())
}
Expand Down
20 changes: 11 additions & 9 deletions core/task/manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -596,16 +596,21 @@ func (m *Manager) acquireTasks(envId uid.ID, taskDescriptors Descriptors) (err e
break DEPLOYMENT_ATTEMPTS_LOOP
}

log.WithField("partition", envId).Errorf("Deployment failed %d/%d attempts. Check messages in IL to figure out why. Retrying...", attemptCount+1, MAX_ATTEMPTS_PER_DEPLOY_REQUEST)
log.WithField("partition", envId).
WithField("level", infologger.IL_Devel).
Errorf("Deployment failed %d/%d attempts. Check messages in IL to figure out why. Retrying...", attemptCount+1, MAX_ATTEMPTS_PER_DEPLOY_REQUEST)
time.Sleep(time.Second * SLEEP_LENGTH_BETWEEN_PER_DEPLOY_REQUESTS)
}
}

logDescriptors("critical task deployment impossible: ", logWithId.Errorf, undeployableCriticalDescriptors)
logDescriptors("critical task deployment failure: ", logWithId.Errorf, undeployedCriticalDescriptors)
{
logWithIdDev := logWithId.WithField("level", infologger.IL_Devel)
logDescriptors("critical task deployment impossible: ", logWithIdDev.Errorf, undeployableCriticalDescriptors)
logDescriptors("critical task deployment failure: ", logWithIdDev.Errorf, undeployedCriticalDescriptors)

logDescriptors("non-critical task deployment failure: ", logWithId.Warningf, undeployedNonCriticalDescriptors)
logDescriptors("non-critical task deployment impossible: ", logWithId.Warningf, undeployableNonCriticalDescriptors)
logDescriptors("non-critical task deployment failure: ", logWithIdDev.Warningf, undeployedNonCriticalDescriptors)
logDescriptors("non-critical task deployment impossible: ", logWithIdDev.Warningf, undeployableNonCriticalDescriptors)
}

// After retries notify environment about failed critical tasks
for _, desc := range undeployableDescriptors {
Expand All @@ -617,11 +622,8 @@ func (m *Manager) acquireTasks(envId uid.ID, taskDescriptors Descriptors) (err e
m.deployMu.Unlock()

if !deploymentSuccess {
// While all the required roles are running, for some reason we
// can't lock some of them, so we must roll back and keep them
// unlocked in the roster.
var deployedTaskIds []string
for taskPtr, _ := range deployedTasks {
for taskPtr := range deployedTasks {
taskPtr.SetParent(nil)
deployedTaskIds = append(deployedTaskIds, taskPtr.taskId)
}
Expand Down
1 change: 1 addition & 0 deletions core/task/scheduler.go
Original file line number Diff line number Diff line change
Expand Up @@ -592,6 +592,7 @@ func (state *schedulerState) resourceOffers(fidStore store.Singleton) events.Han
descriptorsUndeployable = append(descriptorsUndeployable, descriptor)
descriptorsStillToDeploy = append(descriptorsStillToDeploy[:i], descriptorsStillToDeploy[i+1:]...)
log.WithField("partition", envId.String()).
WithField("level", infologger.IL_Devel).
WithField("descriptor", descriptor.TaskClassName).
Errorf("no resource offer for required host %s, deployment will be aborted", requiredMachineId)
}
Expand Down

0 comments on commit f03e42c

Please sign in to comment.