Merge branch 'ipam-service' into 'develop'

Ipam service

See merge request !9

gso/oss-params-example.json

@@ -11,13 +11,20 @@
       "username": "robot-user",
       "password": "robot-user-password"
     },
+    "LO": {
+      "V4": {"containers": ["1.1.0.0/24"], "mask": 32},
+      "V6": {"containers": ["dead:beef::/64"], "mask": 128},
+      "domain_name": ".lo"
+    },
     "TRUNK": {
-      "V4": {"container": "1.1.0.0/24", "mask": 31},
-      "V6": {"container": "dead:beef::/64", "mask": 96}
+      "V4": {"containers": ["1.1.1.0/24"], "mask": 31},
+      "V6": {"containers": ["dead:beef::/64"], "mask": 126},
+      "domain_name": ".trunk"
     },
     "GEANT_IP": {
-      "V4": {"container": "1.1.8.0/24", "mask": 31},
-      "V6": {"container": "dead:beef::/64", "mask": 96}
+      "V4": {"containers": ["1.1.2.0/24"], "mask": 31},
+      "V6": {"containers": ["dead:beef::/64"], "mask": 126},
+      "domain_name": ".geantip"
     }
   },
   "PROVISIONING_PROXY": {

gso/services/ipam.py

 import ipaddress
 from pydantic import BaseSettings
-from gso import settings
+
+from gso.services import _ipam
+
+
+class V4ServiceNetwork(BaseSettings):
+    v4: ipaddress.IPv4Network
+
+
+class V6ServiceNetwork(BaseSettings):
+    v6: ipaddress.IPv6Network
 
 
 class ServiceNetworks(BaseSettings):
@@ -8,26 +17,97 @@ class ServiceNetworks(BaseSettings):
     v6: ipaddress.IPv6Network
 
 
+class V4HostAddress(BaseSettings):
+    v4: ipaddress.IPv4Address
+
+
+class V6HostAddress(BaseSettings):
+    v6: ipaddress.IPv6Address
+
+
 class HostAddresses(BaseSettings):
     v4: ipaddress.IPv4Address
     v6: ipaddress.IPv6Address
 
 
-def new_service_networks(
-        service_params: settings.ServiceNetworkParams) -> ServiceNetworks:
-    oss = settings.load_oss_params()
-    assert oss.IPAM.INFOBLOX
-    # TODO: load from ipam
-    # cf. https://gitlab.geant.org/goat/gap-jenkins/-/blob/development/service-editor/gap_service_editor/ipam.py#L35-66  # noqa: E501
+def new_service_networks(service_type,
+                         comment="",
+                         extattrs={}) -> ServiceNetworks:
+    v4_service_network = _ipam.allocate_service_ipv4_network(
+        service_type=service_type, comment=comment, extattrs=extattrs)
+    v6_service_network = _ipam.allocate_service_ipv6_network(
+        service_type=service_type, comment=comment, extattrs=extattrs)
     return ServiceNetworks(
-        v4=ipaddress.IPv4Network('10.0.0.0/24'),
-        v6=ipaddress.IPv6Network('dead:beef::/120'))
+        v4=v4_service_network.v4,
+        v6=v6_service_network.v6)
+
+
+def new_service_host(hostname,
+                     service_type,
+                     service_networks: ServiceNetworks = None,
+                     host_addresses: HostAddresses = None,
+                     extattrs={}) -> HostAddresses:
+    return _ipam.allocate_service_host(
+        hostname=hostname,
+        service_type=service_type,
+        service_networks=service_networks,
+        host_addresses=host_addresses,
+        extattrs=extattrs)
+
+
+'''
+if __name__ == '__main__':
+    # sample call flow to allocate two loopback interfaces and a trunk service
+    # new_service_host can be called passing networks or addresses
+    # - host h1 for service LO uses a specific ipv4/ipv6 address pair
+    # - the rest use the ipv4/ipv6 network pair
+    # networks and hosts can be allocated with extensible attributes
+    # - host h2 for service LO uses extattrs for both network and address/DNS
+    # - the rest don't use extattrs
+
+    hostname_A = 'h1'
+    hostname_B = 'h2'
+
+    # h1 LO (loopback)
+    lo1_service_networks = new_service_networks(
+        service_type='LO',
+        comment="Network for h1 LO"
+    )
+    lo1_v4_host_address = lo1_service_networks.v4.network_address
+    lo1_v6_host_address = lo1_service_networks.v6.network_address
+    print(lo1_v4_host_address)
+    print(lo1_v6_host_address)
+    lo1_host_addresses = HostAddresses(v4=lo1_v4_host_address,
+                                       v6=lo1_v6_host_address)
+    new_service_host(hostname=hostname_A,
+                     service_type='LO',
+                     host_addresses=lo1_host_addresses)
 
+    # h2 LO (loopback)
+    lo2_network_extattrs = {
+        "vrf_name": {"value": "dummy_vrf"},
+    }
+    lo2_host_extattrs = {
+        "Site": {"value": "dummy_site"},
+    }
+    lo2_service_networks = \
+        new_service_networks(service_type='LO',
+                             comment="Network for h2 LO",
+                             extattrs=lo2_network_extattrs)
+    new_service_host(hostname=hostname_B,
+                     service_type='LO',
+                     service_networks=lo2_service_networks,
+                     extattrs=lo2_host_extattrs)
 
-def new_device_lo_address() -> HostAddresses:
-    oss = settings.load_oss_params()
-    assert oss.IPAM.INFOBLOX
-    # TODO: load from ipam
-    return HostAddresses(
-        v4=ipaddress.IPv4Address('10.10.10.10'),
-        v6=ipaddress.IPv6Address('fc00:798:aa:1::10'))
+    # h1-h2 TRUNK
+    trunk12_service_networks = new_service_networks(
+        service_type='TRUNK',
+        comment="Network for h1-h2 TRUNK"
+    )
+    new_service_host(hostname=hostname_A,
+                     service_type='TRUNK',
+                     service_networks=trunk12_service_networks)
+    new_service_host(hostname=hostname_B,
+                     service_type='TRUNK',
+                     service_networks=trunk12_service_networks)
+'''

gso/settings.py

@@ -17,22 +17,24 @@ class InfoBloxParams(BaseSettings):
 
 
 class V4NetworkParams(BaseSettings):
-    container: ipaddress.IPv4Network
+    containers: list[ipaddress.IPv4Network]
     mask: int  # TODO: validation on mask?
 
 
 class V6NetworkParams(BaseSettings):
-    container: ipaddress.IPv6Network
+    containers: list[ipaddress.IPv6Network]
     mask: int  # TODO: validation on mask?
 
 
 class ServiceNetworkParams(BaseSettings):
     V4: V4NetworkParams
     V6: V6NetworkParams
+    domain_name: str
 
 
 class IPAMParams(BaseSettings):
     INFOBLOX: InfoBloxParams
+    LO: ServiceNetworkParams
     TRUNK: ServiceNetworkParams
     GEANT_IP: ServiceNetworkParams
 

requirements.txt

 orchestrator-core==1.0.0
+pydantic
 requests
 
 pytest
+responses

setup.py

@@ -9,7 +9,8 @@ setup(
     url=('https://gitlab.geant.org/goat/geant-service-orchestrator'),
     packages=find_packages(),
     install_requires=[
-        'requests',
         'orchestrator-core==1.0.0',
+        'pydantic',
+        'requests',
     ]
 )

test/conftest.py

+import contextlib
+import json
+import os
+import socket
+import pytest
+import tempfile
+
+
+@pytest.fixture(scope='session')
+def configuration_data():
+    with contextlib.closing(
+            socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:
+        s.bind(('', 0))
+        s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+        yield {
+            "GENERAL": {
+                "public_hostname": "https://gap.geant.org"
+            },
+            "RESOURCE_MANAGER_API_PREFIX": "http://localhost:44444",
+            "IPAM": {
+                "INFOBLOX": {
+                    "scheme": "https",
+                    "wapi_version": "v2.12",
+                    "host": "10.0.0.1",
+                    "username": "robot-user",
+                    "password": "robot-user-password"
+                },
+                "LO": {
+                    "V4": {"containers": ["10.255.255.0/24"], "mask": 32},
+                    "V6": {"containers": ["dead:beef::/64"], "mask": 128},
+                    "domain_name": ".lo"
+                },
+                "TRUNK": {
+                    "V4": {
+                        "containers": ["10.255.255.0/24", "10.255.254.0/24"],
+                        "mask": 31
+                    },
+                    "V6": {
+                        "containers": ["dead:beef::/64", "dead:beee::/64"],
+                        "mask": 126
+                    },
+                    "domain_name": ".trunk"
+                },
+                "GEANT_IP": {
+                    "V4": {
+                        "containers": ["10.255.255.0/24", "10.255.254.0/24"],
+                        "mask": 31
+                    },
+                    "V6": {
+                        "containers": ["dead:beef::/64", "dead:beee::/64"],
+                        "mask": 126
+                    },
+                    "domain_name": ".geantip"
+                }
+            },
+            "PROVISIONING_PROXY": {
+                "scheme": "https",
+                "api_base": "localhost:44444",
+                "auth": "Bearer <token>",
+                "api_version": 1123
+            }
+        }
+
+
+@pytest.fixture(scope='session')
+def data_config_filename(configuration_data):
+    file_name = os.path.join(
+        tempfile.gettempdir(), os.urandom(24).hex())
+    open(file_name, 'x').close()
+    with open(file_name, 'wb') as f:
+        f.write(json.dumps(configuration_data).encode('utf-8'))
+        f.flush()
+
+        os.environ['OSS_PARAMS_FILENAME'] = f.name
+
+        yield f.name

test/test_ipam.py

+import ipaddress
+import re
+import responses
+
+from gso.services import ipam
+
+
+@responses.activate
+def test_new_service_networks(data_config_filename):
+
+    responses.add(
+        method=responses.POST,
+        url=re.compile(r'.*/wapi.*/network.*'),
+        json={
+            '_ref': 'network/ZG5zLm5ldHdvcmskMTAuMjU1LjI1NS4yMC8zMi8w:10.255.255.20/32/default',   # noqa: E501
+            'network': '10.255.255.20/32'
+        }
+    )
+
+    responses.add(
+        method=responses.POST,
+        url=re.compile(r'.*/wapi.*/ipv6network.*'),
+        json={
+            '_ref': 'ipv6network/ZG5zLm5ldHdvcmskZGVhZDpiZWVmOjoxOC8xMjgvMA:dead%3Abeef%3A%3A18/128/default',   # noqa: E501
+            'network': 'dead:beef::18/128'
+        }
+    )
+
+    service_networks = ipam.new_service_networks(service_type='LO')
+    assert service_networks == ipam.ServiceNetworks(
+        v4=ipaddress.ip_network('10.255.255.20/32'),
+        v6=ipaddress.ip_network('dead:beef::18/128')
+    )
+
+
+@responses.activate
+def test_new_service_host(data_config_filename):
+
+    responses.add(
+        method=responses.POST,
+        url=re.compile(r'.*/wapi.*/record:host$'),
+        json='record:host/ZG5zLmhvc3QkLm5vbl9ETlNfaG9zdF9yb290LjAuMTY4MzcwNTU4MzY3MC5nc28udGVzdA:test.lo/%20'   # noqa: E501
+    )
+
+    responses.add(
+        method=responses.POST,
+        url=re.compile(r'.*/wapi.*/record:a$'),
+        json='record:a/ZG5zLmJpbmRfYSQuX2RlZmF1bHQuZ3NvLHRlc3QsMTAuMjU1LjI1NS44:test.lo/default'   # noqa: E501
+    )
+
+    responses.add(
+        method=responses.POST,
+        url=re.compile(r'.*/wapi.*/record:aaaa$'),
+        json='record:aaaa/ZG5zLmJpbmRfYSQuX2RlZmF1bHQuZ3NvLHRlc3QsMTAuMjU1LjI1NS44:test.lo/default'   # noqa: E501
+    )
+
+    responses.add(
+        method=responses.GET,
+        url=re.compile(r'.*/wapi.*/network.*'),
+        json=[
+            {
+                "_ref": "network/ZG5zLm5ldHdvcmskMTAuMjU1LjI1NS4yMC8zMi8w:10.255.255.20/32/default",   # noqa: E501
+                "network": "10.255.255.20/32",
+                "network_view": "default"
+            }
+        ]
+
+    )
+
+    responses.add(
+        method=responses.GET,
+        url=re.compile(r'.*/wapi.*/ipv6network.*'),
+        json=[
+            {
+                "_ref": "ipv6network/ZG5zLm5ldHdvcmskZGVhZDpiZWVmOjoxOC8xMjgvMA:dead%3Abeef%3A%3A18/128/default",   # noqa: E501
+                "network": "dead:beef::18/128",
+                "network_view": "default"
+            }
+        ]
+    )
+
+    responses.add(
+        method=responses.POST,
+        url=re.compile(r'.*/wapi.*/network.*/.*?_function=next_available_ip&num=1.*'),   # noqa: E501
+        json={'ips': ['10.255.255.20']}
+    )
+
+    responses.add(
+        method=responses.POST,
+        url=re.compile(r'.*/wapi.*/ipv6network.*/.*?_function=next_available_ip&num=1.*'),   # noqa: E501
+        json={'ips': ['dead:beef::18']}
+    )
+
+    service_hosts = ipam.new_service_host(
+        hostname='test',
+        service_type='LO',
+        host_addresses=ipam.HostAddresses(
+            v4=ipaddress.ip_address('10.255.255.20'),
+            v6=ipaddress.ip_address('dead:beef::18')
+        )
+    )
+    assert service_hosts == ipam.HostAddresses(
+        v4=ipaddress.ip_address('10.255.255.20'),
+        v6=ipaddress.ip_address('dead:beef::18')
+    )
+
+    service_hosts = ipam.new_service_host(
+        hostname='test',
+        service_type='LO',
+        service_networks=ipam.ServiceNetworks(
+            v4=ipaddress.ip_network('10.255.255.20/32'),
+            v6=ipaddress.ip_network('dead:beef::18/128')
+        )
+    )
+    assert service_hosts == ipam.HostAddresses(
+        v4=ipaddress.ip_address('10.255.255.20'),
+        v6=ipaddress.ip_address('dead:beef::18')
+    )

test/test_placeholder.py

-# just a placeholder to be able to run tests during ci
-
-def test_placeholder():
-    pass