Merge pull request apache#1483 from remibergsma/pr1413-wilder-47

CLOUDSTACK-9287 - Fix unique mac address per rVPC routerThis is work by @wilderrodrigues, see PR apache#1413 It contains important fixes and I think it needs to be included so I send the PR again.

* pr/1483:
  CLOUDSTACK-9287 - Improve test by checking if pvt gw is removed and fix typos
  CLOUDSTACK-9287 - Fix RVR public interface
  CLOUDSTACK-9287 - Add integration test to cover the private gateway related changes
  CLOUDSTACK-9287 - Refactor the interface state configuration
  CLOUDSTACK-9287 - Check if the nic profile has already been removed from a certain router
  CLOUDSTACK-9287 - Bring up the private gw interface on state change to master
  CLOUDSTACK-9287 - Make sure private gw interface is not used for default gw
  CLOUDSTACK-9287 - Add integration test to cover the private gw interface/mac address issues
  CLOUDSTACK-9287 - Put private gateway interface down on backup router
  CLOUDSTACK-9287 - Generate new mac address if router is redundant and nic profile exists

Signed-off-by: Will Stevens <[email protected]>

server/src/com/cloud/network/element/VirtualRouterElement.java

@@ -24,18 +24,6 @@
 
 import javax.inject.Inject;
 
-import org.apache.cloudstack.api.command.admin.router.ConfigureOvsElementCmd;
-import org.apache.cloudstack.api.command.admin.router.ConfigureVirtualRouterElementCmd;
-import org.apache.cloudstack.api.command.admin.router.CreateVirtualRouterElementCmd;
-import org.apache.cloudstack.api.command.admin.router.ListOvsElementsCmd;
-import org.apache.cloudstack.api.command.admin.router.ListVirtualRouterElementsCmd;
-import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
-import org.apache.cloudstack.network.topology.NetworkTopology;
-import org.apache.cloudstack.network.topology.NetworkTopologyContext;
-import org.apache.log4j.Logger;
-import org.cloud.network.router.deployment.RouterDeploymentDefinition;
-import org.cloud.network.router.deployment.RouterDeploymentDefinitionBuilder;
-
 import com.cloud.agent.api.to.LoadBalancerTO;
 import com.cloud.configuration.ConfigurationManager;
 import com.cloud.dc.DataCenter;
@@ -107,6 +95,18 @@
 import com.cloud.vm.dao.UserVmDao;
 import com.google.gson.Gson;
 
+import org.apache.cloudstack.api.command.admin.router.ConfigureOvsElementCmd;
+import org.apache.cloudstack.api.command.admin.router.ConfigureVirtualRouterElementCmd;
+import org.apache.cloudstack.api.command.admin.router.CreateVirtualRouterElementCmd;
+import org.apache.cloudstack.api.command.admin.router.ListOvsElementsCmd;
+import org.apache.cloudstack.api.command.admin.router.ListVirtualRouterElementsCmd;
+import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
+import org.apache.cloudstack.network.topology.NetworkTopology;
+import org.apache.cloudstack.network.topology.NetworkTopologyContext;
+import org.apache.log4j.Logger;
+import org.cloud.network.router.deployment.RouterDeploymentDefinition;
+import org.cloud.network.router.deployment.RouterDeploymentDefinitionBuilder;
+
 public class VirtualRouterElement extends AdapterBase implements VirtualRouterElementService, DhcpServiceProvider, UserDataServiceProvider, SourceNatServiceProvider,
 StaticNatServiceProvider, FirewallServiceProvider, LoadBalancingServiceProvider, PortForwardingServiceProvider, RemoteAccessVPNServiceProvider, IpDeployer,
 NetworkMigrationResponder, AggregatedCommandExecutor {
@@ -153,6 +153,8 @@ public class VirtualRouterElement extends AdapterBase implements VirtualRouterEl
  IPAddressDao _ipAddressDao;
  @Inject
  DataCenterDao _dcDao;
+ @Inject
+ NetworkModel _networkModel;
 
  @Inject
  NetworkTopologyContext networkTopologyContext;

server/src/com/cloud/network/element/VpcVirtualRouterElement.java

@@ -25,13 +25,6 @@
 
 import javax.inject.Inject;
 
-import org.apache.cloudstack.network.topology.NetworkTopology;
-import org.apache.log4j.Logger;
-import org.cloud.network.router.deployment.RouterDeploymentDefinition;
-import org.cloud.network.router.deployment.RouterDeploymentDefinitionBuilder;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Qualifier;
-
 import com.cloud.dc.DataCenter;
 import com.cloud.dc.DataCenterVO;
 import com.cloud.deploy.DeployDestination;
@@ -79,6 +72,13 @@
 import com.cloud.vm.VirtualMachineManager;
 import com.cloud.vm.VirtualMachineProfile;
 
+import org.apache.cloudstack.network.topology.NetworkTopology;
+import org.apache.log4j.Logger;
+import org.cloud.network.router.deployment.RouterDeploymentDefinition;
+import org.cloud.network.router.deployment.RouterDeploymentDefinitionBuilder;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+
 public class VpcVirtualRouterElement extends VirtualRouterElement implements VpcProvider, Site2SiteVpnServiceProvider, NetworkACLServiceProvider {
 
  private static final Logger s_logger = Logger.getLogger(VpcVirtualRouterElement.class);
@@ -466,7 +466,7 @@ public boolean deletePrivateGateway(final PrivateGateway gateway) throws Concurr
  }
  }
 
- return result > 0 ? true : false;
+ return result == routers.size() ? true : false;
  }
 
  @Override
@@ -559,9 +559,16 @@ public boolean applyACLItemsToPrivateGw(final PrivateGateway gateway, final List
  final DataCenterVO dcVO = _dcDao.findById(network.getDataCenterId());
  final NetworkTopology networkTopology = networkTopologyContext.retrieveNetworkTopology(dcVO);
 
+ final Network privateNetwork = _networkModel.getNetwork(gateway.getNetworkId());
+
  boolean result = true;
  for (final DomainRouterVO domainRouterVO : routers) {
- result = result && networkTopology.applyNetworkACLs(network, rules, domainRouterVO, isPrivateGateway);
+ final NicProfile nicProfile = _networkModel.getNicProfile(domainRouterVO, privateNetwork.getId(), null);
+ if (nicProfile != null) {
+ result = result && networkTopology.applyNetworkACLs(network, rules, domainRouterVO, isPrivateGateway);
+ } else {
+ s_logger.warn("Nic Profile for router '" + domainRouterVO + "' has already been removed. Router is redundant = " + domainRouterVO.getIsRedundantRouter());
+ }
  }
  return result;
  }

server/src/com/cloud/network/router/CommandSetupHelper.java

@@ -58,6 +58,7 @@
 import com.cloud.agent.api.to.IpAddressTO;
 import com.cloud.agent.api.to.LoadBalancerTO;
 import com.cloud.agent.api.to.NetworkACLTO;
+import com.cloud.agent.api.to.NicTO;
 import com.cloud.agent.api.to.PortForwardingRuleTO;
 import com.cloud.agent.api.to.StaticNatRuleTO;
 import com.cloud.agent.manager.Commands;
@@ -504,7 +505,8 @@ public void createNetworkACLsCommands(final List<? extends NetworkACLItem> rules
  }
  }
 
- final SetNetworkACLCommand cmd = new SetNetworkACLCommand(rulesTO, _networkHelper.getNicTO(router, guestNetworkId, null));
+ NicTO nicTO = _networkHelper.getNicTO(router, guestNetworkId, null);
+ final SetNetworkACLCommand cmd = new SetNetworkACLCommand(rulesTO, nicTO);
  cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, _routerControlHelper.getRouterControlIp(router.getId()));
  cmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, _routerControlHelper.getRouterIpInNetwork(guestNetworkId, router.getId()));
  cmd.setAccessDetail(NetworkElementCommand.GUEST_VLAN_TAG, guestVlan);

server/src/com/cloud/network/router/NicProfileHelperImpl.java

@@ -85,6 +85,11 @@ public NicProfile createPrivateNicProfileForGateway(final VpcGateway privateGate
  new NicProfile(privateNic, privateNetwork, privateNic.getBroadcastUri(), privateNic.getIsolationUri(), _networkModel.getNetworkRate(
  privateNetwork.getId(), router.getId()), _networkModel.isSecurityGroupSupportedInNetwork(privateNetwork), _networkModel.getNetworkTag(
  router.getHypervisorType(), privateNetwork));
+
+ if (router.getIsRedundantRouter()) {
+ String newMacAddress = NetUtils.long2Mac(NetUtils.createSequenceBasedMacAddress(ipVO.getMacAddress()));
+ privateNicProfile.setMacAddress(newMacAddress);
+ }
  } else {
  final String netmask = NetUtils.getCidrNetmask(privateNetwork.getCidr());
  final PrivateIpAddress ip =

server/src/com/cloud/network/router/VpcVirtualNetworkApplianceManagerImpl.java

@@ -26,9 +26,6 @@
 import javax.inject.Inject;
 import javax.naming.ConfigurationException;
 
-import org.apache.log4j.Logger;
-import org.springframework.stereotype.Component;
-
 import com.cloud.agent.api.Answer;
 import com.cloud.agent.api.Command;
 import com.cloud.agent.api.Command.OnError;
@@ -91,6 +88,9 @@
 import com.cloud.vm.VirtualMachineProfile.Param;
 import com.cloud.vm.dao.VMInstanceDao;
 
+import org.apache.log4j.Logger;
+import org.springframework.stereotype.Component;
+
 @Component
 public class VpcVirtualNetworkApplianceManagerImpl extends VirtualNetworkApplianceManagerImpl implements VpcVirtualNetworkApplianceManager {
  private static final Logger s_logger = Logger.getLogger(VpcVirtualNetworkApplianceManagerImpl.class);
@@ -531,16 +531,18 @@ protected boolean setupVpcPrivateNetwork(final VirtualRouter router, final boole
 
  @Override
  public boolean destroyPrivateGateway(final PrivateGateway gateway, final VirtualRouter router) throws ConcurrentOperationException, ResourceUnavailableException {
+ boolean result = true;
 
  if (!_networkModel.isVmPartOfNetwork(router.getId(), gateway.getNetworkId())) {
  s_logger.debug("Router doesn't have nic for gateway " + gateway + " so no need to removed it");
- return true;
+ return result;
  }
 
  final Network privateNetwork = _networkModel.getNetwork(gateway.getNetworkId());
+ final NicProfile nicProfile = _networkModel.getNicProfile(router, privateNetwork.getId(), null);
 
  s_logger.debug("Releasing private ip for gateway " + gateway + " from " + router);
- boolean result = setupVpcPrivateNetwork(router, false, _networkModel.getNicProfile(router, privateNetwork.getId(), null));
+ result = setupVpcPrivateNetwork(router, false, nicProfile);
  if (!result) {
  s_logger.warn("Failed to release private ip for gateway " + gateway + " on router " + router);
  return false;
@@ -706,7 +708,7 @@ public boolean startRemoteAccessVpn(final RemoteAccessVpn vpn, final VirtualRout
  s_logger.error("Unable to start vpn: unable add users to vpn in zone " + router.getDataCenterId() + " for account " + vpn.getAccountId() + " on domR: "
  + router.getInstanceName() + " due to " + answer.getDetails());
  throw new ResourceUnavailableException("Unable to start vpn: Unable to add users to vpn in zone " + router.getDataCenterId() + " for account " + vpn.getAccountId()
-  + " on domR: " + router.getInstanceName() + " due to " + answer.getDetails(), DataCenter.class, router.getDataCenterId());
+ + " on domR: " + router.getInstanceName() + " due to " + answer.getDetails(), DataCenter.class, router.getDataCenterId());
  }
  answer = cmds.getAnswer("startVpn");
  if (!answer.getResult()) {

server/src/com/cloud/network/vpc/NetworkACLManagerImpl.java

@@ -21,11 +21,6 @@
 
 import javax.inject.Inject;
 
-import org.apache.cloudstack.context.CallContext;
-import org.apache.cloudstack.framework.messagebus.MessageBus;
-import org.apache.cloudstack.framework.messagebus.PublishScope;
-import org.apache.log4j.Logger;
-
 import com.cloud.configuration.ConfigurationManager;
 import com.cloud.event.ActionEvent;
 import com.cloud.event.EventTypes;
@@ -52,6 +47,11 @@
 import com.cloud.utils.db.TransactionStatus;
 import com.cloud.utils.exception.CloudRuntimeException;
 
+import org.apache.cloudstack.context.CallContext;
+import org.apache.cloudstack.framework.messagebus.MessageBus;
+import org.apache.cloudstack.framework.messagebus.PublishScope;
+import org.apache.log4j.Logger;
+
 public class NetworkACLManagerImpl extends ManagerBase implements NetworkACLManager {
  private static final Logger s_logger = Logger.getLogger(NetworkACLManagerImpl.class);
 
@@ -335,10 +335,10 @@ public boolean revokeACLItemsForNetwork(final long networkId) throws ResourceUna
 
  @Override
  public boolean revokeACLItemsForPrivateGw(final PrivateGateway gateway) throws ResourceUnavailableException {
-
- final List<NetworkACLItemVO> aclItems = _networkACLItemDao.listByACL(gateway.getNetworkACLId());
+ final long networkACLId = gateway.getNetworkACLId();
+ final List<NetworkACLItemVO> aclItems = _networkACLItemDao.listByACL(networkACLId);
  if (aclItems.isEmpty()) {
- s_logger.debug("Found no network ACL Items for private gateway  id=" + gateway.getId());
+ s_logger.debug("Found no network ACL Items for private gateway 'id=" + gateway.getId() + "'");
  return true;
  }
 

server/src/org/apache/cloudstack/network/topology/AdvancedNetworkTopology.java

@@ -19,11 +19,6 @@
 
 import java.util.List;
 
-import org.apache.log4j.Logger;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Qualifier;
-import org.springframework.stereotype.Component;
-
 import com.cloud.dc.DataCenter;
 import com.cloud.deploy.DeployDestination;
 import com.cloud.exception.ConcurrentOperationException;
@@ -52,6 +47,11 @@
 import com.cloud.vm.VirtualMachine.State;
 import com.cloud.vm.VirtualMachineProfile;
 
+import org.apache.log4j.Logger;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.stereotype.Component;
+
 @Component
 public class AdvancedNetworkTopology extends BasicNetworkTopology {
 
@@ -223,6 +223,7 @@ public boolean applyNetworkACLs(final Network network, final List<? extends Netw
 
  final NetworkAclsRules aclsRules = new NetworkAclsRules(network, rules, isPrivateGateway);
 
- return applyRules(network, router, typeString, isPodLevelException, podId, failWhenDisconnect, new RuleApplierWrapper<RuleApplier>(aclsRules));
+ final boolean result = applyRules(network, router, typeString, isPodLevelException, podId, failWhenDisconnect, new RuleApplierWrapper<RuleApplier>(aclsRules));
+ return result;
  }
 }

systemvm/patches/debian/config/opt/cloud/bin/configure.py

@@ -733,34 +733,34 @@ def process(self):
 
  #return the VR guest interface ip
  def getGuestIp(self):
- ipr = []
+ interfaces = []
  ipAddr = None
- for ip in self.config.address().get_ips():
- if ip.is_guest():
- ipr.append(ip)
- if len(ipr) > 0:
- ipAddr = sorted(ipr)[-1]
+ for interface in self.config.address().get_interfaces():
+ if interface.is_guest():
+ interfaces.append(interface)
+ if len(interfaces) > 0:
+ ipAddr = sorted(interfaces)[-1]
  if ipAddr:
  return ipAddr.get_ip()
 
  return None
 
  def getDeviceByIp(self, ipa):
- for ip in self.config.address().get_ips():
- if ip.ip_in_subnet(ipa):
- return ip.get_device()
+ for interface in self.config.address().get_interfaces():
+ if interface.ip_in_subnet(ipa):
+ return interface.get_device()
  return None
 
  def getNetworkByIp(self, ipa):
- for ip in self.config.address().get_ips():
- if ip.ip_in_subnet(ipa):
- return ip.get_network()
+ for interface in self.config.address().get_interfaces():
+ if interface.ip_in_subnet(ipa):
+ return interface.get_network()
  return None
 
  def getGatewayByIp(self, ipa):
- for ip in self.config.address().get_ips():
- if ip.ip_in_subnet(ipa):
- return ip.get_gateway()
+ for interface in self.config.address().get_interfaces():
+ if interface.ip_in_subnet(ipa):
+ return interface.get_gateway()
  return None
 
  def portsToString(self, ports, delimiter):