Merge branch 'feature/test-provisioning-proxy' into 'develop'

Make create_device use provisioning proxy

See merge request !10

.gitignore

@@ -4,4 +4,5 @@ __pycache__/
 coverage.xml
 .tox/device_vendor
 .vscode
-venv
\ No newline at end of file
+venv
+oss-params.json

gso/oss-params-example.json

 {
+  "GENERAL": {
+    "public_hostname": "https://gap.geant.org"
+  },
   "RESOURCE_MANAGER_API_PREFIX": "http://localhost:44444",
   "IPAM": {
     "INFOBLOX": {
@@ -18,8 +21,9 @@
     }
   },
   "PROVISIONING_PROXY": {
-    "api_base": "http://localhost:44444",
+    "scheme": "https",
+    "api_base": "localhost:44444",
     "auth": "Bearer <token>",
     "api_version": 1123
   }
-}
\ No newline at end of file
+}

gso/services/provisioning_proxy.py

 import logging
 
-from gso.products.product_types.device \
-    import DeviceBlock
+import requests
+from orchestrator import inputstep
+from orchestrator.config.assignee import Assignee
+from orchestrator.forms import FormPage, ReadOnlyField
+from orchestrator.forms.validators import Accept, Label, LongText
+from orchestrator.types import UUIDstr, State
+
 from gso import settings
-# import requests
+from gso.products.product_types.device import DeviceProvisioning
 
 logger = logging.getLogger(__name__)
 
 
 def provision_node(
-        node_subscription_params: DeviceBlock,
+        node_subscription_params: DeviceProvisioning,
+        process_id: UUIDstr,
         dry_run: bool = True):
     oss = settings.load_oss_params()
     pp_params = oss.PROVISIONING_PROXY
     assert pp_params
 
-    logger.debug(f'[disabled] provisioning node {node_subscription_params}')
+    device_params = node_subscription_params.device
+    callback_url = f'{settings.load_oss_params().GENERAL.public_hostname}' \
+                   f'/api/processes/{process_id}/resume'
+    logger.debug(f'[disabled] provisioning node {device_params}')
+
+    parameters = {
+        'callback': callback_url,
+        'dry_run': dry_run,
+        'device': {
+            'fqdn': device_params.fqdn,
+            'lo_address': {
+                'v4': str(device_params.lo_ipv4_address),
+                'v6': str(device_params.lo_ipv6_address)
+            },
+            'lo_iso_address': device_params.lo_iso_address,
+            'si_ipv4_network': str(device_params.si_ipv4_network),
+            'ias_lt_network': {
+                'v4': str(device_params.ias_lt_ipv4_network),
+                'v6': str(device_params.ias_lt_ipv6_network)
+            },
+            'site_country_code': device_params.site_country_code,
+            'site_city': device_params.site_city,
+            'site_latitude': device_params.site_latitude,
+            'site_longitude': device_params.site_longitude,
+            'snmp_location': device_params.snmp_location,
+            'device_type': node_subscription_params.device_type,
+            'device_vendor': node_subscription_params.device_vendor,
+            'ts_address': device_params.ts_address,
+            'ts_port': device_params.ts_port
+        }
+    }
+
+    post_request = requests.post(
+        f'{pp_params.scheme}://{pp_params.api_base}'
+        f'/api/device',
+        json=parameters)
+    post_request.raise_for_status()
+
+
+@inputstep('Await provisioning proxy results', assignee=Assignee('SYSTEM'))
+def await_pp_results() -> State:
+    class ProvisioningResultPage(FormPage):
+        class Config:
+            title = 'Do NOT click on confirm in this step!'
+
+        warning_label: Label = 'This step relies on an external service to ' \
+                               'send an update to the orchestrator, do not ' \
+                               'interfere with this process please.'
+        pp_run_results: dict = {'state': 'not_ready'}
+        confirm: Accept = Accept('INCOMPLETE')
+
+    result_page = yield ProvisioningResultPage
+
+    return result_page.dict()
+
+
+@inputstep('Confirm provisioning proxy results', assignee=Assignee('SYSTEM'))
+def confirm_pp_results(state: State) -> State:
+    class ConfirmRunPage(FormPage):
+        class Config:
+            title = 'Execution completed, please confirm the results.'
+
+        run_status: str = ReadOnlyField(state['pp_run_results']['status'])
+        run_results: LongText = ReadOnlyField(
+            f"{state['pp_run_results']['output']}")
+        confirm: Accept = Accept('INCOMPLETE')
+
+    yield ConfirmRunPage
 
-    # r = requests.get(
-    #     f'https://{pp_params.api_base}'
-    #     f'/api/version',
-    #     params=node_subscription_params.dict())
-    # r.raise_for_status()
+    return state

gso/settings.py

@@ -4,6 +4,10 @@ import os
 from pydantic import BaseSettings
 
 
+class GeneralParams(BaseSettings):
+    public_hostname: str
+
+
 class InfoBloxParams(BaseSettings):
     scheme: str
     wapi_version: str
@@ -34,12 +38,14 @@ class IPAMParams(BaseSettings):
 
 
 class ProvisioningProxyParams(BaseSettings):
+    scheme: str
     api_base: str
     auth: str  # FIXME: unfinished
     api_version: int
 
 
 class OSSParams(BaseSettings):
+    GENERAL: GeneralParams
     IPAM: IPAMParams
     RESOURCE_MANAGER_API_PREFIX: str  # api prefix
     PROVISIONING_PROXY: ProvisioningProxyParams

gso/workflows/device/create_device.py

@@ -3,16 +3,18 @@ from uuid import uuid4
 
 from orchestrator.forms import FormPage
 from orchestrator.targets import Target
-from orchestrator.workflow import inputstep
-from orchestrator.forms.validators import Accept
 from orchestrator.types import FormGenerator, State
 from orchestrator.types import SubscriptionLifecycle, UUIDstr
 from orchestrator.workflow import done, init, step, workflow
 from orchestrator.workflows.steps import resync, set_status
 from orchestrator.workflows.steps import store_process_subscription
 from orchestrator.workflows.utils import wrap_create_initial_input_form
-from gso.products.product_types import device
-# from gso.services import ipam, provisioning_proxy
+
+from gso.products.product_types.device import DeviceVendor, DeviceInactive, \
+    DeviceProvisioning
+from gso.services import provisioning_proxy
+from gso.services.provisioning_proxy import confirm_pp_results, \
+    await_pp_results
 
 
 def initial_input_form_generator(product_name: str) -> FormGenerator:
@@ -23,40 +25,41 @@ def initial_input_form_generator(product_name: str) -> FormGenerator:
         fqdn: str
         ts_address: ipaddress.IPv4Address
         ts_port: int
-        device_vendor: device.DeviceVendor
+        device_vendor: DeviceVendor
 
     user_input = yield CreateDeviceForm
 
     return user_input.dict()
 
 
-@step("Create subscription")
+@step('Create subscription')
 def create_subscription(product: UUIDstr) -> State:
-    subscription = device.DeviceInactive.from_product_id(product, uuid4())
+    subscription = DeviceInactive.from_product_id(product, uuid4())
 
     return {
-        "subscription": subscription,
-        "subscription_id": subscription.subscription_id,
+        'subscription': subscription,
+        'subscription_id': subscription.subscription_id,
     }
 
 
-@step("Get information from IPAM ")
-def get_info_from_ipam(subscription: device.DeviceInactive) -> State:
+@step('Get information from IPAM')
+def get_info_from_ipam(subscription: DeviceInactive) -> State:
     # lo = ipam.new_device_lo_address()
     # subscription.device.lo_ipv4_address = lo.v4
     # subscription.device.lo_ipv6_address = lo.v6
     # TODO: get info about how these should be generated
-    subscription.device.lo_ipv4_address = "10.10.10.10"
-    subscription.device.lo_ipv6_address = "fc00:798:10::10"
-    subscription.device.lo_iso_address = "49.51e5.0001.0620.4009.6047.00"
-    subscription.device.si_ipv4_network = "192.168.0.0/31"
-    subscription.device.ias_lt_ipv4_network = "192.168.1.0/31"
-    subscription.device.ias_lt_ipv6_network = "fc00:798:1::150/126"
-    return {"subscription": subscription}
+    subscription.device.lo_ipv4_address = '10.10.10.10'
+    subscription.device.lo_ipv6_address = 'fc00:798:10::10'
+    subscription.device.lo_iso_address = '49.51e5.0001.0620.4009.6047.00'
+    subscription.device.si_ipv4_network = '192.168.0.0/31'
+    subscription.device.ias_lt_ipv4_network = '192.168.1.0/31'
+    subscription.device.ias_lt_ipv6_network = 'fc00:798:1::150/126'
+
+    return {'subscription': subscription}
 
 
-@step("get information about SNMP")
-def get_snmp_info(subscription: device.DeviceInactive) -> State:
+@step('Get information about SNMP')
+def get_snmp_info(subscription: DeviceInactive) -> State:
     country = 'Spain'
     city = 'Barcelona'
     country_code = 'ES'
@@ -71,132 +74,74 @@ def get_snmp_info(subscription: device.DeviceInactive) -> State:
         f'{city.upper()},{country.upper()}[{latitude},{longitude}]'
     )
 
-    return {"subscription": subscription}
+    return {'subscription': subscription}
 
 
-@step("Initialize subscription")
+@step('Initialize subscription')
 def initialize_subscription(
-    subscription: device.DeviceInactive,
-    fqdn: str,
-    ts_address: ipaddress.IPv4Address,
-    ts_port: str,
-    device_vendor: device.DeviceVendor
+        subscription: DeviceInactive,
+        fqdn: str,
+        ts_address: ipaddress.IPv4Address,
+        ts_port: str,
+        device_vendor: DeviceVendor
 ) -> State:
     subscription.device.fqdn = fqdn
     subscription.device.ts_address = str(ts_address)
     subscription.device.ts_port = str(ts_port)
     subscription.device_vendor = device_vendor
-    subscription.description = f"Device {fqdn} type \
-                                ({subscription.device_type})"
-    subscription = device.DeviceProvisioning.from_other_lifecycle(
+    subscription.description = f'Device {fqdn} ' \
+                               f'({subscription.device_type})'
+    subscription = DeviceProvisioning.from_other_lifecycle(
         subscription, SubscriptionLifecycle.PROVISIONING
     )
 
-    return {"subscription": subscription}
+    return {'subscription': subscription}
 
 
-@step("Provision device [DRY RUN]")
-def provision_device_dry(
-    subscription: device.DeviceProvisioning,
-    fqdn: str,
-    ts_address: str,
-    ts_port: str
-) -> State:
-    # import ansible_runner
-    #
-    # r = ansible_runner.run(
-    #     private_data_dir="/opt/geant-gap-ansible",
-    #     playbook="base_config.yaml",
-    #     inventory=subscription.device.fqdn,
-    #     extravars={
-    #         "lo_ipv4_address": str(subscription.device.lo_ipv4_address),
-    #         "lo_ipv6_address": str(subscription.device.lo_ipv6_address),
-    #         "lo_iso_address": subscription.device.lo_iso_address,
-    #         "snmp_location": subscription.device.snmp_location,
-    #         "si_ipv4_network": str(subscription.device.si_ipv4_network),
-    #         "lt_ipv4_network": str(subscription.device.ias_lt_ipv4_network),
-    #         "lt_ipv6_network": str(subscription.device.ias_lt_ipv6_network),
-    #         "site_country_code": subscription.device.site_country_code,
-    #         "verb": "deploy",
-    #     },
-    # )
-    # out = r.stdout.read()
-    # out_splitted = out.splitlines()
-    # # if r.rc != 0:
-    # #    raise ValueError("Ansible has failed")
-    # return {"dry_run_output": out_splitted, "return_code": r.rc}
-    # provisioning_proxy.provision_node(
-    #     node_subscription_params=subscription,
-    #     dry_run=True)
-    #  TODO: figure out what to return when we are suspending & waiting
-    #        for the provisioning-proxy to call back
-    return {"return_code": 0}
-
-
-@inputstep("Confirm step", assignee="CHANGES")
-def confirm_step() -> FormGenerator:
-    class ConfirmForm(FormPage):
-        confirm: Accept
-
-    user_input = yield ConfirmForm
-
-    return {"confirm": user_input.confirm}
-
-
-@step("Provision device [FOR REAL]")
-def provision_device_real(
-    subscription: device.DeviceProvisioning,
-    fqdn: str,
-    ts_address: str,
-    ts_port: str
-) -> State:
-    # import ansible_runner
-    #
-    # r = ansible_runner.run(
-    #     private_data_dir="/opt/geant-gap-ansible",
-    #     playbook="base_config.yaml",
-    #     inventory=subscription.device.fqdn,
-    #     extravars={
-    #         "lo_ipv4_address": str(subscription.device.lo_ipv4_address),
-    #         "lo_ipv6_address": str(subscription.device.lo_ipv6_address),
-    #         "lo_iso_address": subscription.device.lo_iso_address,
-    #         "snmp_location": subscription.device.snmp_location,
-    #         "si_ipv4_network": str(subscription.device.si_ipv4_network),
-    #         "lt_ipv4_network": str(subscription.device.ias_lt_ipv4_network),
-    #         "lt_ipv6_network": str(subscription.device.ias_lt_ipv6_network),
-    #         "site_country_code": subscription.device.site_country_code,
-    #         "verb": "deploy",
-    #     },
-    # )
-    # out = r.stdout.read()
-    # out_splitted = out.splitlines()
-    #
-    # return {"real_run_output": out_splitted, "return_code": r.rc}
-    # provisioning_proxy.provision_node(
-    #    node_subscription_params=subscription)
-    # TODO: figure out what to return when we are suspending & waiting
-    #       for the provisioning-proxy to call back
-    return {"return_code": 0}
+@step('Provision device [DRY RUN]')
+def provision_device_dry(subscription: DeviceProvisioning,
+                         process_id: UUIDstr) -> State:
+    provisioning_proxy.provision_node(
+        subscription,
+        process_id
+    )
+
+    return {'subscription': subscription}
+
+
+@step('Provision device [FOR REAL]')
+def provision_device_real(subscription: DeviceProvisioning,
+                          process_id: UUIDstr) -> State:
+    provisioning_proxy.provision_node(
+        subscription,
+        process_id,
+        False  # No dry run this time, run it for real
+    )
+
+    return {'subscription': subscription}
 
 
 @workflow(
-    "Create Device",
+    'Create device',
     initial_input_form=wrap_create_initial_input_form(
-                       initial_input_form_generator),
+        initial_input_form_generator),
     target=Target.CREATE,
 )
 def create_device():
     return (
-        init
-        >> create_subscription
-        >> store_process_subscription(Target.CREATE)
-        >> get_info_from_ipam
-        >> get_snmp_info
-        >> initialize_subscription
-        >> provision_device_dry
-        >> confirm_step
-        >> provision_device_real
-        >> set_status(SubscriptionLifecycle.ACTIVE)
-        >> resync
-        >> done
+            init
+            >> create_subscription
+            >> store_process_subscription(Target.CREATE)
+            >> get_info_from_ipam
+            >> get_snmp_info
+            >> initialize_subscription
+            >> provision_device_dry
+            >> await_pp_results
+            >> confirm_pp_results
+            >> provision_device_real
+            >> await_pp_results
+            >> confirm_pp_results
+            >> set_status(SubscriptionLifecycle.ACTIVE)
+            >> resync
+            >> done
     )

tox.ini

@@ -15,4 +15,3 @@ commands =
     # coverage report --fail-under 80
     coverage report
     flake8
-