NAT-152: Add support from network rather than container in oss configuration to allocate hosts from

gso/services/_ipam.py

@@ -37,6 +37,8 @@ class HostAddresses(BaseSettings):
 class IPAMErrors(Enum):
     # HTTP error code, match in error message
     CONTAINER_FULL = 400, "Can not find requested number of networks"
+    NETWORK_FULL = 400, \
+        "Cannot find 1 available IP address(es) in this network"
     EXTATTR_UNKNOWN = 400, "Unknown extensible attribute"
     EXTATTR_BADVALUE = 400, "Bad value for extensible attribute"
 
@@ -121,6 +123,10 @@ def _allocate_network(
     ip_container = 'networkcontainer' if ip_version == 4 else \
         'ipv6networkcontainer'
 
+    assert network_params.containers, \
+        "No containers available to allocate networks for this service." \
+        "Maybe you want to allocate a host from a network directly?"
+
     # only return in the response the allocated network, not all available
     # TODO: any validation needed for extrattrs wherever it's used?
     req_payload = {
@@ -206,13 +212,22 @@ def allocate_service_ipv6_network(service_type, comment="", extattrs={}
 
 
 def _find_next_available_ip(infoblox_params, network_ref):
+    """
+    Find the next available IP address from a network given its ref.
+    Returns "NETWORK_FULL" if there's no space in the network.
+    Otherwise returns the next available IP address in the network.
+    """
     r = requests.post(
         f'{_wapi(infoblox_params)}/{network_ref}?_function=next_available_ip&num=1',   # noqa: E501
         auth=HTTPBasicAuth(infoblox_params.username,
                            infoblox_params.password),
         verify=False
     )
-    # TODO: propagate no more available IPs in the network
+
+    if _match_error_code(response=r,
+                         error_code=IPAMErrors.NETWORK_FULL):
+        return "NETWORK_FULL"
+
     assert r.status_code >= 200 and r.status_code < 300, \
         f"HTTP error {r.status_code}: {r.reason}\n\n{r.text}"
     assert 'ips' in r.json()
@@ -226,11 +241,12 @@ def _allocate_host(hostname=None,
                    networks=None,
                    cname_aliases=None,
                    extattrs={}
-                   ) -> HostAddresses:
+                   ) -> Union[HostAddresses, str]:
     """
     If networks is not None, allocate host in those networks.
     Otherwise if addrs is not None, allocate host with those addresses.
     hostname parameter must be full name including domain name.
+    Return an error string if couldn't allocate host due to network full.
     """
     # TODO: should hostnames be unique
     # (i.e. fail if hostname already exists in this domain/service)?
@@ -261,6 +277,13 @@ def _allocate_host(hostname=None,
         ipv6_addr = _find_next_available_ip(infoblox_params,
                                             network_info[0]["_ref"])
 
+        # If couldn't find next available IPs, return error
+        if ipv4_addr == "NETWORK_FULL" or ipv6_addr == "NETWORK_FULL":
+            if ipv4_addr == "NETWORK_FULL":
+                return "IPV4_NETWORK_FULL"
+            if ipv6_addr == "NETWORK_FULL":
+                return "IPV6_NETWORK_FULL"
+
     else:
         ipv4_addr = addrs[0]
         ipv6_addr = addrs[1]
@@ -332,12 +355,17 @@ def allocate_service_host(hostname=None,
     """
     Allocate host record with both IPv4 and IPv6 address, and respective DNS
     A and AAAA records.
-    - If service_networks is provided, that one is used.
+    - If service_networks is provided, and it's in a valid container,
+    that one is used.
     - If service_networks is not provided, and host_addresses is provided,
     those specific addresses are used.
-    - If neither is not provided, new ipv4 and ipv6 networks are created and
-    those are used. Note that in this case extattrs is for the hosts and not
-    for the networks.
+    - If neither is provided:
+        - If service has configured containers, new ipv4 and ipv6 networks are
+        created and those are used. Note that in this case extattrs is for the
+        hosts and not for the networks.
+        - If service doesn't have configured containers and has configured
+        networks instead, the configured networks are used (they are filled up
+        in order of appearance in the configuration file).
     The domain name is taken from the service type and appended to the
     specified hostname.
     """
@@ -347,42 +375,81 @@ def allocate_service_host(hostname=None,
 
     assert hasattr(ipam_params, service_type) \
         and service_type != 'INFOBLOX', "Invalid service type."
-    ipv4_containers = getattr(ipam_params, service_type).V4.containers
-    ipv6_containers = getattr(ipam_params, service_type).V6.containers
+    oss_ipv4_containers = getattr(ipam_params, service_type).V4.containers
+    oss_ipv6_containers = getattr(ipam_params, service_type).V6.containers
+    oss_ipv4_networks = getattr(ipam_params, service_type).V4.networks
+    oss_ipv6_networks = getattr(ipam_params, service_type).V6.networks
     domain_name = getattr(ipam_params, service_type).domain_name
 
+    assert (oss_ipv4_containers and oss_ipv6_containers) \
+        or (oss_ipv4_networks and oss_ipv6_networks), \
+        "This service is missing either containers or networks configuration."
+    assert domain_name, "This service is missing domain_name configuration."
+
     if cname_aliases:
         cname_aliases = [alias + domain_name for alias in cname_aliases]
 
     if not service_networks and not host_addresses:
-        # IPv4
-        ipv4_network = str(allocate_service_ipv4_network(
-                       service_type=service_type).v4)
-        assert ipv4_network, \
-            "No available space for IPv4 networks for this service type."
-
-        # IPv6
-        ipv6_network = str(allocate_service_ipv6_network(
-                       service_type=service_type).v6)
-        assert ipv6_network, \
-            "No available space for IPv6 networks for this service type."
-
-        network_tuple = (ipv4_network, ipv6_network)
-        host = _allocate_host(hostname=hostname+domain_name,
-                              networks=network_tuple,
-                              cname_aliases=cname_aliases,
-                              extattrs=extattrs)
+        if oss_ipv4_containers and oss_ipv6_containers:
+        # This service has configured containers.
+        # Use them to allocate new networks that can allocate the hosts.
+
+            # IPv4
+            ipv4_network = str(allocate_service_ipv4_network(
+                           service_type=service_type).v4)
+            assert ipv4_network, \
+                "No available space for IPv4 networks for this service type."
+
+            # IPv6
+            ipv6_network = str(allocate_service_ipv6_network(
+                           service_type=service_type).v6)
+            assert ipv6_network, \
+                "No available space for IPv6 networks for this service type."
+
+        elif oss_ipv4_networks and oss_ipv6_networks:
+        # This service has configured networks.
+        # Allocate a host inside an ipv4 and ipv6 network from among them.
+            ipv4_network = str(oss_ipv4_networks[0])
+            ipv6_network = str(oss_ipv6_networks[0])
+
+        while True:
+            ipv4_network_index = 0
+            ipv6_network_index = 0
+            network_tuple = (ipv4_network, ipv6_network)
+            host = _allocate_host(hostname=hostname+domain_name,
+                                  networks=network_tuple,
+                                  cname_aliases=cname_aliases,
+                                  extattrs=extattrs)
+
+            if "NETWORK_FULL" not in host:
+                break
+            elif "IPV4" in host:
+                ipv4_network_index += 1
+                assert ipv4_network_index < len(oss_ipv4_networks), \
+                    "No available space in any IPv4 network for this service."
+                ipv4_network = str(oss_ipv4_networks[ipv4_network_index])
+            else: # IPV6 in host
+                ipv6_network_index += 1
+                assert ipv6_network_index < len(oss_ipv6_networks), \
+                    "No available space in any IPv6 network for this service."
+                ipv6_network = str(oss_ipv6_networks[ipv6_network_index])
 
     elif service_networks:
         # IPv4
         ipv4_network = service_networks.v4
-        assert any(ipv4_network.subnet_of(ipv4_container)
-                   for ipv4_container in ipv4_containers)
+        if oss_ipv4_containers:
+            assert any(ipv4_network.subnet_of(oss_ipv4_container)
+                       for oss_ipv4_container in oss_ipv4_containers)
+        else:
+            assert ipv4_network in oss_ipv4_networks
 
         # IPv6
         ipv6_network = service_networks.v6
-        assert any(ipv6_network.subnet_of(ipv6_container)
-                   for ipv6_container in ipv6_containers)
+        if oss_ipv6_containers:
+            assert any(ipv6_network.subnet_of(oss_ipv6_container)
+                       for oss_ipv6_container in oss_ipv6_containers)
+        else:
+            assert ipv6_network in oss_ipv6_networks
 
         host = _allocate_host(
             hostname=hostname+domain_name,
@@ -390,17 +457,26 @@ def allocate_service_host(hostname=None,
             cname_aliases=cname_aliases,
             extattrs=extattrs
         )
+        assert "NETWORK_FULL" not in host
 
     elif host_addresses:
         # IPv4
         ipv4_addr = host_addresses.v4
-        assert any(ipv4_addr in ipv4_container
-                   for ipv4_container in ipv4_containers)
+        if oss_ipv4_containers:
+            assert any(ipv4_addr in oss_ipv4_container
+                       for oss_ipv4_container in oss_ipv4_containers)
+        else:
+            assert any(ipv4_addr in oss_ipv4_network
+                       for oss_ipv4_network in oss_ipv4_networks)
 
         # IPv6
         ipv6_addr = host_addresses.v6
-        assert any(ipv6_addr in ipv6_container
-                   for ipv6_container in ipv6_containers)
+        if oss_ipv6_containers:
+            assert any(ipv6_addr in oss_ipv6_container
+                       for oss_ipv6_container in oss_ipv6_containers)
+        else:
+            assert any(ipv4_addr in oss_ipv6_network
+                       for oss_ipv6_network in oss_ipv6_networks)
 
         host = _allocate_host(
             hostname=hostname+domain_name,
@@ -408,6 +484,7 @@ def allocate_service_host(hostname=None,
             cname_aliases=cname_aliases,
             extattrs=extattrs
         )
+        assert "NETWORK_FULL" not in host
 
     return host
 

gso/services/ipam.py

@@ -57,61 +57,54 @@ def new_service_host(hostname,
         extattrs=extattrs)
 
 
-'''
+
 if __name__ == '__main__':
     # sample call flow to allocate two loopback interfaces and a trunk service
-    # new_service_host can be called passing networks or addresses
-    # - host h1 for service LO uses a specific ipv4/ipv6 address pair
-    # - the rest use the ipv4/ipv6 network pair
+    # new_service_host can be called passing networks, addresses, or nothing.
+    # - host h1 for service TRUNK uses a specific ipv4/ipv6 address pair
+    # - host h2 for service TRUNK uses the ipv4/ipv6 network pair
+    # - service LO uses nothing
     # networks and hosts can be allocated with extensible attributes
-    # - host h2 for service LO uses extattrs for both network and address/DNS
-    # - the rest don't use extattrs
+    # networks can be created with a comment
     # CNAME records can be optionally created
-    # - hosts for service TRUNK use some alias names
-
-    hostname_A = 'h1'
-    hostname_B = 'h2'
 
+    hostname_A = 'h9'
+    hostname_B = 'h10'
+    '''
     # h1 LO (loopback)
-    lo1_service_networks = new_service_networks(
-        service_type='LO',
-        comment="Network for h1 LO"
-    )
-    lo1_v4_host_address = lo1_service_networks.v4.network_address
-    lo1_v6_host_address = lo1_service_networks.v6.network_address
-    lo1_host_addresses = HostAddresses(v4=lo1_v4_host_address,
-                                       v6=lo1_v6_host_address)
     new_service_host(hostname=hostname_A+"_LO",
-                     service_type='LO',
-                     host_addresses=lo1_host_addresses)
+                     service_type='LO')
 
     # h2 LO (loopback)
-    lo2_network_extattrs = {
+    new_service_host(hostname=hostname_B+"_LO",
+                     service_type='LO')
+    '''
+    # h1-h2 TRUNK
+    trunk12_network_extattrs = {
         "vrf_name": {"value": "dummy_vrf"},
     }
-    lo2_host_extattrs = {
+    trunk12_host_extattrs = {
         "Site": {"value": "dummy_site"},
     }
-    lo2_service_networks = \
-        new_service_networks(service_type='LO',
-                             comment="Network for h2 LO",
-                             extattrs=lo2_network_extattrs)
-    new_service_host(hostname=hostname_B+"_LO",
-                     service_type='LO',
-                     service_networks=lo2_service_networks,
-                     extattrs=lo2_host_extattrs)
-
-    # h1-h2 TRUNK
     trunk12_service_networks = new_service_networks(
         service_type='TRUNK',
+        extattrs=trunk12_network_extattrs,
         comment="Network for h1-h2 TRUNK"
     )
+
+    trunk12_v4_host_address = trunk12_service_networks.v4.network_address
+    trunk12_v6_host_address = trunk12_service_networks.v6.network_address
+    trunk12_host_addresses = HostAddresses(v4=trunk12_v4_host_address,
+                                           v6=trunk12_v6_host_address)
+
     new_service_host(hostname=hostname_A+"_TRUNK",
                      service_type='TRUNK',
+                     host_addresses=trunk12_host_addresses,
                      cname_aliases=["alias1.h1", "alias2.h1"],
-                     service_networks=trunk12_service_networks)
+                     extattrs=trunk12_host_extattrs)
+
     new_service_host(hostname=hostname_B+"_TRUNK",
                      service_type='TRUNK',
+                     service_networks=trunk12_service_networks,
                      cname_aliases=["alias1.h2"],
-                     service_networks=trunk12_service_networks)
-'''
+                     extattrs=trunk12_host_extattrs)

gso/settings.py

@@ -18,11 +18,13 @@ class InfoBloxParams(BaseSettings):
 
 class V4NetworkParams(BaseSettings):
     containers: list[ipaddress.IPv4Network]
+    networks: list[ipaddress.IPv4Network]
     mask: int  # TODO: validation on mask?
 
 
 class V6NetworkParams(BaseSettings):
     containers: list[ipaddress.IPv6Network]
+    networks: list[ipaddress.IPv6Network]
     mask: int  # TODO: validation on mask?