update workflows to make use of new provisioning proxy pattern with 3 retries

gso/services/provisioning_proxy.py

@@ -7,13 +7,14 @@ import logging
 from typing import NoReturn
 
 import requests
-from orchestrator import inputstep
+from orchestrator import conditional, inputstep, step
 from orchestrator.config.assignee import Assignee
 from orchestrator.domain import SubscriptionModel
 from orchestrator.forms import FormPage, ReadOnlyField
 from orchestrator.forms.validators import Accept, Label, LongText
 from orchestrator.types import FormGenerator, State, UUIDstr, strEnum
 from orchestrator.utils.json import json_dumps
+from orchestrator.workflow import Step, StepList
 from pydantic import validator
 
 from gso import settings
@@ -162,8 +163,15 @@ def await_pp_results(subscription: SubscriptionModel, label_text: str = DEFAULT_
     return result_page.dict()
 
 
+@step("Reset Provisioning Proxy state")
+def reset_pp_success_state() -> State:
+    return {"pp_did_succeed": False}
+
+
 @inputstep("Confirm provisioning proxy results", assignee=Assignee("SYSTEM"))
 def confirm_pp_results(state: State) -> FormGenerator:
+    successful_run = state["pp_run_results"]["return_code"] == 0
+
     class ConfirmRunPage(FormPage):
         class Config:
             title = (
@@ -174,9 +182,22 @@ def confirm_pp_results(state: State) -> FormGenerator:
 
         run_status: str = ReadOnlyField(state["pp_run_results"]["status"])
         run_results: LongText = ReadOnlyField(f"{state['pp_run_results']['output']}")
-        pp_result_good_enough: bool = state["pp_run_results"]["return_code"] == 0
+        if not successful_run:
+            pp_retry_label: Label = (
+                "Provisioning Proxy playbook execution failed, it will be retried (up to two times)."  # type: ignore
+            )
         confirm: Accept = Accept("INCOMPLETE")
 
-    confirmation = yield ConfirmRunPage
+    yield ConfirmRunPage
+
+    return {"pp_did_succeed": successful_run}
+
+
+def pp_interaction(provisioning_step: Step) -> StepList:
+    should_retry_pp_steps = conditional(lambda state: not state.get("pp_did_succeed"))
 
-    return {"pp_did_succeed": confirmation.pp_result_good_enough}
+    return (
+        should_retry_pp_steps(provisioning_step)
+        >> should_retry_pp_steps(await_pp_results)
+        >> should_retry_pp_steps(confirm_pp_results)
+    )