From 7f2ccb2f18650503df90d132979bdba5c66c852a Mon Sep 17 00:00:00 2001 From: Alexander Verhaar Date: Fri, 20 Jul 2018 20:54:59 +0200 Subject: [PATCH] Fix problem with hanging HV and HA --- .../cloud/agent/manager/AgentManagerImpl.java | 129 +++++--- .../VirtualNetworkApplianceManagerImpl.java | 285 ++++++++---------- .../com/cloud/legacymodel/dc/HostStatus.java | 3 + .../com/cloud/model/enumeration/Event.java | 3 +- 4 files changed, 222 insertions(+), 198 deletions(-) diff --git a/cosmic-core/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java b/cosmic-core/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java index 21b62a86d0..307f3ba121 100644 --- a/cosmic-core/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java +++ b/cosmic-core/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java @@ -12,8 +12,10 @@ import com.cloud.db.model.Zone; import com.cloud.db.repository.ZoneRepository; import com.cloud.dc.ClusterVO; +import com.cloud.dc.DataCenterVO; import com.cloud.dc.HostPodVO; import com.cloud.dc.dao.ClusterDao; +import com.cloud.dc.dao.DataCenterDao; import com.cloud.dc.dao.HostPodDao; import com.cloud.framework.config.ConfigKey; import com.cloud.framework.config.Configurable; @@ -74,6 +76,9 @@ import com.cloud.utils.nio.NioServer; import com.cloud.utils.nio.Task; import com.cloud.utils.time.InaccurateClock; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.slf4j.MDC; import javax.inject.Inject; import javax.naming.ConfigurationException; @@ -95,10 +100,6 @@ import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.slf4j.MDC; - /** * Implementation of the Agent Manager. This class controls the connection to the agents. **/ @@ -146,6 +147,8 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl @Inject protected HostDao _hostDao = null; @Inject + protected DataCenterDao _dcDao = null; + @Inject protected HostPodDao _podDao = null; @Inject protected ConfigurationDao _configDao = null; @@ -422,6 +425,7 @@ public void handleCommands(final AgentAttache attache, final long sequence, fina protected boolean handleDisconnectWithInvestigation(final AgentAttache attache, Event event) { final long hostId = attache.getId(); HostVO host = this._hostDao.findById(hostId); + boolean removeAgent = false; if (host != null) { HostStatus nextStatus = null; @@ -435,72 +439,94 @@ protected boolean handleDisconnectWithInvestigation(final AgentAttache attache, s_logger.debug("Caught exception while getting agent's next status", ne); } + // For log and alert purposes later + final DataCenterVO dcVO = _dcDao.findById(host.getDataCenterId()); + final HostPodVO podVO = _podDao.findById(host.getPodId()); + final String hostDesc = "[name: " + host.getName() + " (id:" + host.getId() + "), availability zone: " + dcVO.getName() + ", pod: " + podVO.getName() + "]"; + final String hostShortDesc = "Host " + host.getName() + " (id:" + host.getId() + ")"; + + final ResourceState resourceState = host.getResourceState(); + if (resourceState == ResourceState.Disabled || resourceState == ResourceState.Maintenance || resourceState == ResourceState.ErrorInMaintenance) { + // If we are in this resourceState, no need to investigate or do anything. AgentMonitor will handle when in these resourceStates + s_logger.info(hostShortDesc + " has disconnected with event " + event + ", but is in Resource State of " + resourceState + ", so doing nothing"); + return true; + } + if (nextStatus == HostStatus.Alert) { /* OK, we are going to the bad status, let's see what happened */ - s_logger.info("Investigating why host " + hostId + " has disconnected with event " + event); + s_logger.info("Investigating why host " + hostShortDesc + " has disconnected with event " + event); HostStatus determinedState = investigate(attache); // if state cannot be determined do nothing and bail out if (determinedState == null) { if ((System.currentTimeMillis() >> 10) - host.getLastPinged() > this.AlertWait.value()) { - s_logger.warn("Agent " + hostId + " state cannot be determined for more than " + this.AlertWait + "(" + this.AlertWait.value() + ") seconds, will go to Alert state"); + s_logger.warn("State for " + hostShortDesc + " could not be determined for more than " + AlertWait + "(" + AlertWait.value() + ") seconds, will go to Alert state"); determinedState = HostStatus.Alert; } else { - s_logger.warn("Agent " + hostId + " state cannot be determined, do nothing"); + s_logger.warn("State for " + hostShortDesc + " could not be determined, doing nothing"); return false; } } final HostStatus currentStatus = host.getStatus(); - s_logger.info("The agent from host " + hostId + " state determined is " + determinedState); + s_logger.info("Status for " + hostShortDesc + " was " + currentStatus + ". Investigation determined the current state is " + determinedState); - if (determinedState == HostStatus.Down) { - final String message = "Host is down: " + host.getId() + "-" + host.getName() + ". Starting HA on the VMs"; - s_logger.error(message); - if (host.getType() != HostType.SecondaryStorage && host.getType() != HostType.ConsoleProxy) { - this._alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_HOST, host.getDataCenterId(), host.getPodId(), "Host down, " + host.getId(), message); - } - event = Event.HostDown; - } else if (determinedState == HostStatus.Up) { + if (determinedState == HostStatus.Up) { /* Got ping response from host, bring it back */ - s_logger.info("Agent is determined to be up and running"); + s_logger.info(hostShortDesc + " is up again"); agentStatusTransitTo(host, Event.Ping, this._nodeId); - return false; } else if (determinedState == HostStatus.Disconnected) { - s_logger.warn("Agent is disconnected but the host is still up: " + host.getId() + "-" + host.getName()); + // Investigation says host isn't down, just disconnected if (currentStatus == HostStatus.Disconnected) { if ((System.currentTimeMillis() >> 10) - host.getLastPinged() > this.AlertWait.value()) { - s_logger.warn("Host " + host.getId() + " has been disconnected past the wait time it should be disconnected."); + s_logger.error("The agent on " + hostShortDesc + " has been disconnected longer than " + AlertWait + " (" + AlertWait.value() + " seconds). Setting event to WaitedTooLong"); + if (host.getType() != HostType.SecondaryStorage && host.getType() != HostType.ConsoleProxy) { + _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_HOST, host.getDataCenterId(), host.getPodId(), hostShortDesc + " is in Alert status", + "The agent for host " + hostDesc + " has been disconnected longer than " + AlertWait + " (" + AlertWait.value() + " seconds), host will be put into Alert status."); + } event = Event.WaitedTooLong; } else { - s_logger.debug("Host " + host.getId() + " has been determined to be disconnected but it hasn't passed the wait time yet."); + s_logger.warn(hostShortDesc + " has been disconnected for " + ((System.currentTimeMillis() >> 10) - host.getLastPinged()) + " seconds, but for less than " + AlertWait + " (" + AlertWait.value() + " seconds). No action taken."); return false; } } else if (currentStatus == HostStatus.Up) { - final Zone zone = this._zoneRepository.findById(host.getDataCenterId()).orElse(null); - final HostPodVO podVO = this._podDao.findById(host.getPodId()); - final String hostDesc = "name: " + host.getName() + " (id:" + host.getId() + "), availability zone: " + zone.getName() + ", pod: " + podVO.getName(); - if (host.getType() != HostType.SecondaryStorage && host.getType() != HostType.ConsoleProxy) { - this._alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_HOST, host.getDataCenterId(), host.getPodId(), "Host disconnected, " + hostDesc, - "If the agent for host [" + hostDesc + "] is not restarted within " + this.AlertWait + " seconds, host will go to Alert state"); - } - event = Event.AgentDisconnected; + s_logger.warn(hostShortDesc + " was up but is now disconnected. Setting event to AgentUnreachable."); + event = Event.AgentUnreachable; + _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_HOST, host.getDataCenterId(), host.getPodId(), hostShortDesc + " is " + determinedState, + "The host " + hostDesc + " was " + currentStatus + ", but has now the host agent is in " + HostStatus.Disconnected + ". If the host is disconnected longer than " + AlertWait + " (" + AlertWait.value() + " seconds), it will be put into Alert status."); + } else if (currentStatus == HostStatus.Alert) { + s_logger.error(hostShortDesc + " was in and is still in " + HostStatus.Alert + ". No action taken."); + return false; + } else { + // If we are here, host was in another status, but next status is alert so let's alert + s_logger.error(hostShortDesc + " was in " + currentStatus + ", and investigation determined it is " + HostStatus.Disconnected); + _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_HOST, host.getDataCenterId(), host.getPodId(), hostShortDesc + " is " + determinedState, "The host " + hostDesc + " was in " + currentStatus + ", and investigation determined it is " + HostStatus.Disconnected); + } + } else if (determinedState == HostStatus.Down) { + s_logger.error(hostShortDesc + " is down. Setting event to HostDown and will start HA on the VMs"); + if (host.getType() != HostType.SecondaryStorage && host.getType() != HostType.ConsoleProxy) { + _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_HOST, host.getDataCenterId(), host.getPodId(), hostShortDesc + " is down", "Host " + hostDesc + " is down. Will start HA on the VMs"); + } else { + _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_HOST, host.getDataCenterId(), host.getPodId(), hostShortDesc + " is down", "Host " + hostDesc + " is down."); } + event = Event.HostDown; + removeAgent = true; + } else if (determinedState == HostStatus.Alert) { + // This is likely a Console Proxy or Secondary Storage VM + s_logger.error("Investigation found " + hostShortDesc + " in state: " + determinedState); + _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_HOST, host.getDataCenterId(), host.getPodId(), "Investigation found " + hostShortDesc + " in " + determinedState, "Investigation found " + hostShortDesc + " in " + determinedState + " state. Event is " + event); } else { - // if we end up here we are in alert state, send an alert - final Zone zone = this._zoneRepository.findById(host.getDataCenterId()).orElse(null); - final HostPodVO podVO = this._podDao.findById(host.getPodId()); - final String podName = podVO != null ? podVO.getName() : "NO POD"; - final String hostDesc = "name: " + host.getName() + " (id:" + host.getId() + "), availability zone: " + zone.getName() + ", pod: " + podName; - this._alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_HOST, host.getDataCenterId(), host.getPodId(), "Host in ALERT state, " + hostDesc, - "In availability zone " + host.getDataCenterId() + ", host is in alert state: " + host.getId() + "-" + host.getName()); + // Determined state was not Up, Disconnected, Down, or Alert. To catch anything else create alert + s_logger.error("Investigation found " + hostShortDesc + " in unhandled state: " + determinedState); + _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_HOST, host.getDataCenterId(), host.getPodId(), "Investigation found " + hostShortDesc + " in " + determinedState, "Investigation found " + hostShortDesc + " in unhandled state: " + determinedState); } } else { - s_logger.debug("The next status of agent " + host.getId() + " is not Alert, no need to investigate what happened"); + s_logger.info("The next status of host " + hostShortDesc + " is " + nextStatus + ", no need to investigate what happened"); } } - handleDisconnectWithoutInvestigation(attache, event, true, true); - host = this._hostDao.findById(hostId); // Maybe the host magically reappeared? + + handleDisconnectWithoutInvestigation(attache, event, true, removeAgent); + host = _hostDao.findById(hostId); // We may have transitioned the status - refresh if (host != null && host.getStatus() == HostStatus.Down) { this._haMgr.scheduleRestartForVmsOnHost(host, true); } @@ -537,6 +563,17 @@ protected boolean handleDisconnectWithoutInvestigation(final AgentAttache attach s_logger.warn("Can't find host with " + hostId); nextStatus = HostStatus.Removed; } else { + final ResourceState resourceState = host.getResourceState(); + if (host.getType() == HostType.Routing && event == Event.ShutdownRequested && resourceState != ResourceState.Maintenance && removeAgent) { + // The host agent is shutting itself down gracefully, the host isn't in maintenance and we are removing the host's monitoring task. + final DataCenterVO dcVO = _dcDao.findById(host.getDataCenterId()); + final HostPodVO podVO = _podDao.findById(host.getPodId()); + final String hostDesc = "[name: " + host.getName() + " (id:" + host.getId() + "), availability zone: " + dcVO.getName() + ", pod: " + podVO.getName() + "]"; + final String hostShortDesc = "Host " + host.getName() + " (id:" + host.getId() + ")"; + s_logger.warn(hostShortDesc + " has disconnected with event " + event + ", but was in resource state " + resourceState + ", not in Maintenance. Agent Monitor is also being removed."); + _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_HOST, host.getDataCenterId(), host.getPodId(), hostShortDesc + " agent is shutting down", + "The agent for host " + hostDesc + " has disconnected with event " + event + ", but was in resource state " + resourceState + ", not in Maintenance. Host should be put into Maintenance before shutting down the host agent."); + } final HostStatus currentStatus = host.getStatus(); if (currentStatus == HostStatus.Down || currentStatus == HostStatus.Alert || currentStatus == HostStatus.Removed) { if (s_logger.isDebugEnabled()) { @@ -553,16 +590,17 @@ protected boolean handleDisconnectWithoutInvestigation(final AgentAttache attach } if (s_logger.isDebugEnabled()) { - s_logger.debug("The next status of agent " + hostId + "is " + nextStatus + ", current status is " + currentStatus); + s_logger.debug("The next status of agent " + hostId + " is " + nextStatus + ", current status is " + currentStatus); } } } - if (s_logger.isDebugEnabled()) { - s_logger.debug("Deregistering link for " + hostId + " with state " + nextStatus); + if (removeAgent) { + if (s_logger.isDebugEnabled()) { + s_logger.debug("Deregistering link for " + hostId + " with state " + nextStatus); + } + removeAgent(attache, nextStatus); } - - removeAgent(attache, nextStatus); // update the DB if (host != null && transitState) { disconnectAgent(host, event, this._nodeId); @@ -1244,7 +1282,7 @@ protected void runInContext() { if (this._investigate == true) { handleDisconnectWithInvestigation(this._attache, this._event); } else { - handleDisconnectWithoutInvestigation(this._attache, this._event, true, false); + handleDisconnectWithoutInvestigation(this._attache, this._event, true, true); } } catch (final Exception e) { s_logger.error("Exception caught while handling disconnect: ", e); @@ -1543,6 +1581,7 @@ protected void runInContext() { final String hostDesc = "name: " + host.getName() + " (id:" + host.getId() + "), availability zone: " + zone.getName() + ", pod: " + podVO.getName(); AgentManagerImpl.this._alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_HOST, host.getDataCenterId(), host.getPodId(), "Migration Complete for host " + hostDesc, "Host [" + hostDesc + "] is ready for maintenance"); + disconnectWithoutInvestigation(host.getId(), Event.ShutdownRequested); // Host now in maintenance, disconnect and remove Monitor } } diff --git a/cosmic-core/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java b/cosmic-core/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java index ab8dfa4b90..84fd6782ba 100644 --- a/cosmic-core/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java +++ b/cosmic-core/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java @@ -162,6 +162,9 @@ import com.cloud.vm.dao.DomainRouterDao; import com.cloud.vm.dao.NicDao; import com.cloud.vm.dao.UserVmDetailsDao; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Qualifier; import javax.inject.Inject; import javax.naming.ConfigurationException; @@ -185,10 +188,6 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Qualifier; - /** * VirtualNetworkApplianceManagerImpl manages the different types of virtual * network appliances available in the Cloud Stack. @@ -296,6 +295,7 @@ public class VirtualNetworkApplianceManagerImpl extends ManagerBase implements V ScheduledExecutorService _executor; ScheduledExecutorService _checkExecutor; ScheduledExecutorService _networkStatsUpdateExecutor; + ExecutorService _routerOobStartExecutor; ExecutorService _rvrStatusUpdateExecutor; BlockingQueue _vrUpdateQueue = null; private String _dnsBasicZoneUpdates = "all"; @@ -313,6 +313,7 @@ public boolean configure(final String name, final Map params) th _executor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("RouterMonitor")); _checkExecutor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("RouterStatusMonitor")); _networkStatsUpdateExecutor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("NetworkStatsUpdater")); + _routerOobStartExecutor = Executors.newCachedThreadPool(new NamedThreadFactory("CheckRouterAfterOobStartExecutor")); VirtualMachine.State.getStateMachine().registerListener(this); @@ -1491,7 +1492,7 @@ public void doInTransactionWithoutResult(final TransactionStatus status) { final List routerGuestNtwkIds = _routerDao.getRouterNetworks(router.getId()); for (final Long guestNtwkId : routerGuestNtwkIds) { final UserStatisticsVO userStats = _userStatsDao.lock(router.getAccountId(), router.getDataCenterId(), guestNtwkId, null, router.getId(), router.getType() - .toString()); + .toString()); if (userStats != null) { final long currentBytesRcvd = userStats.getCurrentBytesReceived(); userStats.setCurrentBytesReceived(0); @@ -1539,78 +1540,7 @@ public void prepareStop(final VirtualMachineProfile profile) { } if (forVpc && network.getTrafficType() == TrafficType.Public || !forVpc && network.getTrafficType() == TrafficType.Guest && network.getGuestType() == GuestType.Isolated) { - final NetworkUsageCommand usageCmd = new NetworkUsageCommand(privateIP, router.getHostName(), forVpc, routerNic.getIPv4Address()); - final String routerType = router.getType().toString(); - final UserStatisticsVO previousStats = _userStatsDao.findBy(router.getAccountId(), router.getDataCenterId(), network.getId(), - forVpc ? routerNic.getIPv4Address() : null, router.getId(), routerType); - NetworkUsageAnswer answer = null; - try { - answer = (NetworkUsageAnswer) _agentMgr.easySend(router.getHostId(), usageCmd); - } catch (final Exception e) { - s_logger.warn("Error while collecting network stats from router: " + router.getInstanceName() + " from host: " + router.getHostId(), e); - continue; - } - - if (answer != null) { - if (!answer.getResult()) { - s_logger.warn("Error while collecting network stats from router: " + router.getInstanceName() + " from host: " + router.getHostId() + "; details: " - + answer.getDetails()); - continue; - } - try { - if (answer.getBytesReceived() == 0 && answer.getBytesSent() == 0) { - s_logger.debug("Recieved and Sent bytes are both 0. Not updating user_statistics"); - continue; - } - - final NetworkUsageAnswer answerFinal = answer; - Transaction.execute(new TransactionCallbackNoReturn() { - @Override - public void doInTransactionWithoutResult(final TransactionStatus status) { - final UserStatisticsVO stats = _userStatsDao.lock(router.getAccountId(), router.getDataCenterId(), network.getId(), - forVpc ? routerNic.getIPv4Address() : null, router.getId(), routerType); - if (stats == null) { - s_logger.warn("unable to find stats for account: " + router.getAccountId()); - return; - } - - if (previousStats != null - && (previousStats.getCurrentBytesReceived() != stats.getCurrentBytesReceived() || previousStats.getCurrentBytesSent() != stats - .getCurrentBytesSent())) { - s_logger.debug("Router stats changed from the time NetworkUsageCommand was sent. " + "Ignoring current answer. Router: " - + answerFinal.getRouterName() + " Rcvd: " + answerFinal.getBytesReceived() + "Sent: " + answerFinal.getBytesSent()); - return; - } - - if (stats.getCurrentBytesReceived() > answerFinal.getBytesReceived()) { - if (s_logger.isDebugEnabled()) { - s_logger.debug("Received # of bytes that's less than the last one. " + "Assuming something went wrong and persisting it. Router: " - + answerFinal.getRouterName() + " Reported: " + answerFinal.getBytesReceived() + " Stored: " + stats.getCurrentBytesReceived()); - } - stats.setNetBytesReceived(stats.getNetBytesReceived() + stats.getCurrentBytesReceived()); - } - stats.setCurrentBytesReceived(answerFinal.getBytesReceived()); - if (stats.getCurrentBytesSent() > answerFinal.getBytesSent()) { - if (s_logger.isDebugEnabled()) { - s_logger.debug("Received # of bytes that's less than the last one. " + "Assuming something went wrong and persisting it. Router: " - + answerFinal.getRouterName() + " Reported: " + answerFinal.getBytesSent() + " Stored: " + stats.getCurrentBytesSent()); - } - stats.setNetBytesSent(stats.getNetBytesSent() + stats.getCurrentBytesSent()); - } - stats.setCurrentBytesSent(answerFinal.getBytesSent()); - if (!_dailyOrHourly) { - // update agg bytes - stats.setAggBytesSent(stats.getNetBytesSent() + stats.getCurrentBytesSent()); - stats.setAggBytesReceived(stats.getNetBytesReceived() + stats.getCurrentBytesReceived()); - } - _userStatsDao.update(stats.getId(), stats); - } - }); - } catch (final Exception e) { - s_logger.warn("Unable to update user statistics for account: " + router.getAccountId() + " Rx: " + answer.getBytesReceived() + "; Tx: " - + answer.getBytesSent()); - } - } + updateUsage(privateIP, router, forVpc, routerNic, network); } } } @@ -1778,12 +1708,93 @@ public boolean postStateTransitionEvent(final Transition answerFinal.getBytesReceived()) { + if (s_logger.isDebugEnabled()) { + s_logger.debug("Received # of bytes that's less than the last one. " + "Assuming something went wrong and persisting it. Router: " + + answerFinal.getRouterName() + " Reported: " + answerFinal.getBytesReceived() + " Stored: " + stats.getCurrentBytesReceived()); + } + stats.setNetBytesReceived(stats.getNetBytesReceived() + stats.getCurrentBytesReceived()); + } + stats.setCurrentBytesReceived(answerFinal.getBytesReceived()); + if (stats.getCurrentBytesSent() > answerFinal.getBytesSent()) { + if (s_logger.isDebugEnabled()) { + s_logger.debug("Received # of bytes that's less than the last one. " + "Assuming something went wrong and persisting it. Router: " + + answerFinal.getRouterName() + " Reported: " + answerFinal.getBytesSent() + " Stored: " + stats.getCurrentBytesSent()); + } + stats.setNetBytesSent(stats.getNetBytesSent() + stats.getCurrentBytesSent()); + } + stats.setCurrentBytesSent(answerFinal.getBytesSent()); + if (!_dailyOrHourly) { + // update agg bytes + stats.setAggBytesSent(stats.getNetBytesSent() + stats.getCurrentBytesSent()); + stats.setAggBytesReceived(stats.getNetBytesReceived() + stats.getCurrentBytesReceived()); + } + _userStatsDao.update(stats.getId(), stats); + } + }); + } catch (final Exception e) { + s_logger.warn("Unable to update user statistics for account: " + router.getAccountId() + " Rx: " + answer.getBytesReceived() + "; Tx: " + + answer.getBytesSent()); + } + } + } + private boolean isOutOfBandMigrated(final Object opaque) { // opaque -> if (opaque != null && opaque instanceof Pair) { @@ -1837,80 +1848,7 @@ protected void runInContext() { } if (forVpc && network.getTrafficType() == TrafficType.Public || !forVpc && network.getTrafficType() == TrafficType.Guest && network.getGuestType() == GuestType.Isolated) { - final NetworkUsageCommand usageCmd = new NetworkUsageCommand(privateIP, router.getHostName(), forVpc, routerNic.getIPv4Address()); - final String routerType = router.getType().toString(); - final UserStatisticsVO previousStats = _userStatsDao.findBy(router.getAccountId(), router.getDataCenterId(), network.getId(), - forVpc ? routerNic.getIPv4Address() : null, router.getId(), routerType); - NetworkUsageAnswer answer = null; - try { - answer = (NetworkUsageAnswer) _agentMgr.easySend(router.getHostId(), usageCmd); - } catch (final Exception e) { - s_logger.warn("Error while collecting network stats from router: " + router.getInstanceName() + " from host: " + router.getHostId(), e); - continue; - } - - if (answer != null) { - if (!answer.getResult()) { - s_logger.warn("Error while collecting network stats from router: " + router.getInstanceName() + " from host: " + router.getHostId() - + "; details: " + answer.getDetails()); - continue; - } - try { - if (answer.getBytesReceived() == 0 && answer.getBytesSent() == 0) { - s_logger.debug("Recieved and Sent bytes are both 0. Not updating user_statistics"); - continue; - } - final NetworkUsageAnswer answerFinal = answer; - Transaction.execute(new TransactionCallbackNoReturn() { - @Override - public void doInTransactionWithoutResult(final TransactionStatus status) { - final UserStatisticsVO stats = _userStatsDao.lock(router.getAccountId(), router.getDataCenterId(), network.getId(), - forVpc ? routerNic.getIPv4Address() : null, router.getId(), routerType); - if (stats == null) { - s_logger.warn("unable to find stats for account: " + router.getAccountId()); - return; - } - - if (previousStats != null - && (previousStats.getCurrentBytesReceived() != stats.getCurrentBytesReceived() || previousStats.getCurrentBytesSent() != - stats - .getCurrentBytesSent())) { - s_logger.debug("Router stats changed from the time NetworkUsageCommand was sent. " + "Ignoring current answer. Router: " - + answerFinal.getRouterName() + " Rcvd: " + answerFinal.getBytesReceived() + "Sent: " + answerFinal.getBytesSent()); - return; - } - - if (stats.getCurrentBytesReceived() > answerFinal.getBytesReceived()) { - if (s_logger.isDebugEnabled()) { - s_logger.debug("Received # of bytes that's less than the last one. " - + "Assuming something went wrong and persisting it. Router: " + answerFinal.getRouterName() + " Reported: " - + answerFinal.getBytesReceived() + " Stored: " + stats.getCurrentBytesReceived()); - } - stats.setNetBytesReceived(stats.getNetBytesReceived() + stats.getCurrentBytesReceived()); - } - stats.setCurrentBytesReceived(answerFinal.getBytesReceived()); - if (stats.getCurrentBytesSent() > answerFinal.getBytesSent()) { - if (s_logger.isDebugEnabled()) { - s_logger.debug("Received # of bytes that's less than the last one. " - + "Assuming something went wrong and persisting it. Router: " + answerFinal.getRouterName() + " Reported: " - + answerFinal.getBytesSent() + " Stored: " + stats.getCurrentBytesSent()); - } - stats.setNetBytesSent(stats.getNetBytesSent() + stats.getCurrentBytesSent()); - } - stats.setCurrentBytesSent(answerFinal.getBytesSent()); - if (!_dailyOrHourly) { - // update agg bytes - stats.setAggBytesSent(stats.getNetBytesSent() + stats.getCurrentBytesSent()); - stats.setAggBytesReceived(stats.getNetBytesReceived() + stats.getCurrentBytesReceived()); - } - _userStatsDao.update(stats.getId(), stats); - } - }); - } catch (final Exception e) { - s_logger.warn("Unable to update user statistics for account: " + router.getAccountId() + " Rx: " + answer.getBytesReceived() + "; Tx: " - + answer.getBytesSent()); - } - } + updateUsage(privateIP, router, forVpc, routerNic, network); } } } @@ -1973,6 +1911,49 @@ public void doInTransactionWithoutResult(final TransactionStatus status) { } } + protected class CheckRouterAfterOobStart extends ManagedContextRunnable { + + long _routerId; + + public CheckRouterAfterOobStart(final long routerId) { + _routerId = routerId; + } + + @Override + protected void runInContext() { + /* Since vRouter appears to be powered-on OOB, make sure we can talk to router + * If we can't talk to it, we need to reboot it to get it managed correctly + * This is needed for example when a host agent goes down and comes back up, + * we would have done a failed HA event on the router and end up having our controlIP out-of-sync + */ + final DomainRouterVO router = _routerDao.findById(_routerId); + final CheckRouterCommand command = new CheckRouterCommand(); + final String routerDesc = router.getInstanceName() + " (ID:" + _routerId + ")"; + + command.setAccessDetail(NetworkElementCommand.ROUTER_IP, _routerControlHelper.getRouterControlIp(_routerId)); + command.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); + command.setWait(30); + final Answer answer = _agentMgr.easySend(router.getHostId(), command); + boolean cmdSuccess = false; + if (answer instanceof CheckRouterAnswer) { + if (answer != null) { + if (answer.getResult()) { + s_logger.info("Successfully able to send CheckRouterCommand to " + routerDesc + " after out-of-band power-on"); + cmdSuccess = true; + } + } + } + if (!cmdSuccess) { + s_logger.warn("Unable to send CheckRouterCommand to " + routerDesc + ", rebooting router "); + try { + rebootRouter(_routerId, true); + } catch (final Exception e) { + s_logger.error("Error while rebooting router " + routerDesc, e); + } + } + } + } + protected class RvRStatusUpdateTask extends ManagedContextRunnable { public RvRStatusUpdateTask() { diff --git a/cosmic-model/src/main/java/com/cloud/legacymodel/dc/HostStatus.java b/cosmic-model/src/main/java/com/cloud/legacymodel/dc/HostStatus.java index b0eededefe..a4cebe30ae 100644 --- a/cosmic-model/src/main/java/com/cloud/legacymodel/dc/HostStatus.java +++ b/cosmic-model/src/main/java/com/cloud/legacymodel/dc/HostStatus.java @@ -34,6 +34,7 @@ public enum HostStatus { s_fsm.addTransition(HostStatus.Connecting, Event.AgentDisconnected, HostStatus.Alert); s_fsm.addTransition(HostStatus.Up, Event.PingTimeout, HostStatus.Alert); s_fsm.addTransition(HostStatus.Up, Event.AgentDisconnected, HostStatus.Alert); + s_fsm.addTransition(HostStatus.Up, Event.AgentUnreachable, HostStatus.Disconnected); s_fsm.addTransition(HostStatus.Up, Event.ShutdownRequested, HostStatus.Disconnected); s_fsm.addTransition(HostStatus.Up, Event.HostDown, HostStatus.Down); s_fsm.addTransition(HostStatus.Up, Event.Ping, HostStatus.Up); @@ -56,10 +57,12 @@ public enum HostStatus { s_fsm.addTransition(HostStatus.Down, Event.PingTimeout, HostStatus.Down); s_fsm.addTransition(HostStatus.Alert, Event.AgentConnected, HostStatus.Connecting); s_fsm.addTransition(HostStatus.Alert, Event.Ping, HostStatus.Up); + s_fsm.addTransition(HostStatus.Alert, Event.PingTimeout, HostStatus.Alert); s_fsm.addTransition(HostStatus.Alert, Event.Remove, HostStatus.Removed); s_fsm.addTransition(HostStatus.Alert, Event.ManagementServerDown, HostStatus.Alert); s_fsm.addTransition(HostStatus.Alert, Event.AgentDisconnected, HostStatus.Alert); s_fsm.addTransition(HostStatus.Alert, Event.ShutdownRequested, HostStatus.Disconnected); + s_fsm.addTransition(HostStatus.Alert, Event.HostDown, HostStatus.Down); s_fsm.addTransition(HostStatus.Rebalancing, Event.RebalanceFailed, HostStatus.Disconnected); s_fsm.addTransition(HostStatus.Rebalancing, Event.RebalanceCompleted, HostStatus.Connecting); s_fsm.addTransition(HostStatus.Rebalancing, Event.ManagementServerDown, HostStatus.Disconnected); diff --git a/cosmic-model/src/main/java/com/cloud/model/enumeration/Event.java b/cosmic-model/src/main/java/com/cloud/model/enumeration/Event.java index 9ebe66f796..14b941f51c 100644 --- a/cosmic-model/src/main/java/com/cloud/model/enumeration/Event.java +++ b/cosmic-model/src/main/java/com/cloud/model/enumeration/Event.java @@ -5,10 +5,11 @@ public enum Event { PingTimeout(false, "Agent is behind on ping"), ShutdownRequested(false, "Shutdown requested by the agent"), AgentDisconnected(false, "Agent disconnected"), + AgentUnreachable(false, "Host is found to be disconnected by the investigator"), HostDown(false, "Host is found to be down by the investigator"), Ping(false, "Ping is received from the host"), ManagementServerDown(false, "Management Server that the agent is connected is going down"), - WaitedTooLong(false, "Waited too long from the agent to reconnect on its own. Time to do HA"), + WaitedTooLong(false, "Waited too long from the agent to reconnect on its own."), Remove(true, "Host is removed"), Ready(false, "Host is ready for commands"), RequestAgentRebalance(false, "Request rebalance for the certain host"),