diff --git a/geant/gap_ansible/playbooks/base_config_checks.yaml b/geant/gap_ansible/playbooks/base_config_checks.yaml index a00837e86c2705a0b139a1957197f1b904d8af21..8ee3838b790298a0dacb5c2f77165795388bbb9b 100644 --- a/geant/gap_ansible/playbooks/base_config_checks.yaml +++ b/geant/gap_ansible/playbooks/base_config_checks.yaml @@ -2,7 +2,5 @@ hosts: all connection: netconf gather_facts: false - # remote_user: "{{ gap_ansible_user }}" - remote_user: geant-ne-na-lab roles: - ../roles/base_config_checks diff --git a/geant/gap_ansible/playbooks/check_ibgp.yaml b/geant/gap_ansible/playbooks/check_ibgp.yaml index 53b384096c74f1cb97e0fb9cee9577e29d963507..9f85b5e9f33c7a7713aa3d7e3f2e4d675d8a2dbd 100644 --- a/geant/gap_ansible/playbooks/check_ibgp.yaml +++ b/geant/gap_ansible/playbooks/check_ibgp.yaml @@ -1,7 +1,5 @@ - name: Check iBGP status on P hosts: all - # remote_user: "{{ gap_ansible_user }}" - remote_user: geant-ne-na-lab gather_facts: false roles: - Juniper.junos diff --git a/geant/gap_ansible/playbooks/iptrunks_checks.yaml b/geant/gap_ansible/playbooks/iptrunks_checks.yaml index bc134eb7b5fc45aa2f2e6e5d8b7e9deb41682a37..b171eec5914583422ddbd7fdf22767ec07069e27 100644 --- a/geant/gap_ansible/playbooks/iptrunks_checks.yaml +++ b/geant/gap_ansible/playbooks/iptrunks_checks.yaml @@ -1,7 +1,5 @@ - name: Manage iptrunks checks playbook hosts: all - # remote_user: "{{ gap_ansible_user }}" - # remote_user: geant-ne-na-lab gather_facts: false roles: - Juniper.junos diff --git a/geant/gap_ansible/playbooks/iptrunks_migration.yaml b/geant/gap_ansible/playbooks/iptrunks_migration.yaml index e6b61639f9f27d1e1156f059e2480b622f1270f8..6943bd7baa114493c008788d58eef7a61be55d8d 100644 --- a/geant/gap_ansible/playbooks/iptrunks_migration.yaml +++ b/geant/gap_ansible/playbooks/iptrunks_migration.yaml @@ -1,7 +1,5 @@ - name: Manage iptrunks migration hosts: all - # remote_user: "{{ gap_ansible_user }}" - remote_user: geant-ne-na-lab gather_facts: false roles: - Juniper.junos diff --git a/geant/gap_ansible/playbooks/update_ibgp_mesh.yaml b/geant/gap_ansible/playbooks/update_ibgp_mesh.yaml index 73599f7a91a4eb0439f407cfce86c69d5fbc65a4..947dcc2780667cbce2a3f2db7621c4886e8662b5 100644 --- a/geant/gap_ansible/playbooks/update_ibgp_mesh.yaml +++ b/geant/gap_ansible/playbooks/update_ibgp_mesh.yaml @@ -1,7 +1,5 @@ - name: Manage iBGP mesh hosts: all - # remote_user: "{{ gap_ansible_user }}" - remote_user: geant-ne-na-lab gather_facts: false roles: - Juniper.junos diff --git a/roles/check_demarcation_point/tasks/checks.yaml b/roles/check_demarcation_point/tasks/checks.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ea9820ab4084c25d719e8ef936393a776e9085c0 --- /dev/null +++ b/roles/check_demarcation_point/tasks/checks.yaml @@ -0,0 +1,94 @@ +--- + +- name: Load variables from pre and post check results + ansible.builtin.include_vars: + dir: vars + files_matching: "{{ check_id }}.yml" + +- name: TEST Print PRE interface statistics + ansible.builtin.debug: + var: pre_check + when: verb == 'pre_check' + +- name: TEST Print POST interface statistics + ansible.builtin.debug: + var: post_check + when: verb == 'post_check' + +- name: Get the interface status + ansible.builtin.assert: + that: + - "{{ verb }}.{{ check_id }}.interface_status == 'up'" + fail_msg: "FAIL: physical interface is DOWN" + success_msg: "PASS: physical interface is UP" + failed_when: false + +- name: Check the interface input errors + ansible.builtin.assert: + that: + - "{{ verb }}.{{ check_id }}.input_error_count <= {{ input_error_count_threshold }}" + fail_msg: "FAIL: input errors detected" + success_msg: "PASS: no input errors detected" + failed_when: false + + +- name: Check the interface output errors + ansible.builtin.assert: + that: + - "{{ verb }}.{{ check_id }}.output_error_count <= {{ output_error_count_threshold }}" + fail_msg: "FAIL: output errors detected" + success_msg: "PASS: no output errors detected" + failed_when: false + +- name: Check the interface FEC statistics + ansible.builtin.assert: + that: + - "{{ verb }}.{{ check_id }}.ethernet_fec_statistics.fec_ccw_count is defined" + - "{{ verb }}.{{ check_id }}.ethernet_fec_statistics.fec_ccw_count.0.data <= {{ fec_ccw_count_threshold }}" + - "{{ verb }}.{{ check_id }}.ethernet_fec_statistics.fec_ccw_error_rate.0.data == {{ fec_ccw_error_rate_threshold }}" + - "{{ verb }}.{{ check_id }}.ethernet_fec_statistics.fec_nccw_count.0.data == {{ fec_nccw_count_threshold }}" + - "{{ verb }}.{{ check_id }}.ethernet_fec_statistics.fec_nccw_error_rate.0.data == {{ fec_nccw_error_rate_threshold }}" + fail_msg: "FAIL: FEC statistics errors detected or FEC statistics not found - please check results in relevant yaml file" + success_msg: "PASS: no FEC statistics errors detected" + failed_when: false + +- name: Check the interface PCS statistics + ansible.builtin.assert: + that: + - "{{ verb }}.{{ check_id }}.ethernet_pcs_statistics_bit_error_secs | int <= {{ pcs_bit_error_seconds_threshold }}" + - "{{ verb }}.{{ check_id }}.ethernet_pcs_statistics_errored_blocks_secs | int <= {{ pcs_bit_error_blocks_seconds_threshold }}" + fail_msg: "FAIL: PCS statistics errors detected" + success_msg: "PASS: PCS statistics errors are below threshold" + failed_when: false + +- name: Check the optical levels output Tx power low threshold (dBm) + ansible.builtin.assert: + that: + - "{{ verb }}.{{ check_id }}.output_power_dbm > {{ min_tx_power_dbm }}" + fail_msg: "FAIL: Low Tx output power detected (dBm)" + success_msg: "PASS: Tx Output power above minimum threshold (dBm) " + failed_when: false + +- name: Check the optical levels output Tx power high threshold (dBm) + ansible.builtin.assert: + that: + - "{{ verb }}.{{ check_id }}.output_power_dbm < {{ max_tx_power_dbm }}" + fail_msg: "FAIL: High Tx output power detected (dBm)" + success_msg: "PASS: Tx Output power below maximum threshold (dBm) " + failed_when: false + +- name: Check the optical levels input Rx power low threshold (dBm) + ansible.builtin.assert: + that: + - "{{ verb }}.{{ check_id }}.rx_signal_avg_power_dbm > {{ min_rx_power_dbm }}" + fail_msg: "FAIL: Low Rx signal power detected (dBm)" + success_msg: "PASS: Rx signal power above minimum threshold (dBm) " + failed_when: false + +- name: Check the optical levels input Rx power high threshold (dBm) + ansible.builtin.assert: + that: + - "{{ verb }}.{{ check_id }}.rx_signal_avg_power_dbm < {{ max_rx_power_dbm }}" + fail_msg: "FAIL: High Rx signal power detected (dBm)" + success_msg: "PASS: Rx signal power below maximum threshold (dBm) " + failed_when: false diff --git a/roles/check_demarcation_point/tasks/compare.yaml b/roles/check_demarcation_point/tasks/compare.yaml new file mode 100644 index 0000000000000000000000000000000000000000..790eeba19ed43b8b1dd635306b1deb775c2c564a --- /dev/null +++ b/roles/check_demarcation_point/tasks/compare.yaml @@ -0,0 +1,111 @@ +--- + +- name: Get output file information + ansible.builtin.stat: + path: roles/check_demarcation_point/vars/{{ check_id }}.yml + register: file_status + +- name: Load variables from pre and post check results + ansible.builtin.include_vars: + dir: vars + files_matching: "{{ check_id }}.yml" + +- name: Check if PRE and POST measurements exist prior to compare command + ansible.builtin.assert: + that: + - "pre_check.{{ check_id }} is defined" + - "post_check.{{ check_id }} is defined" + fail_msg: "FAIL: PRE and POST measurements have not been performed before COMPARE" + success_msg: "PASS: PRE and POST values exist " + +- name: Compare PRE and POST interface status + ansible.builtin.assert: + that: + - "pre_check.{{ check_id }}.interface_status == post_check.{{ check_id }}.interface_status" + fail_msg: "FAIL: Interface status on PRE and post POST is not equal" + success_msg: "PASS: Interface status PRE and POST check is equal " + +- name: Compare PRE and POST interface input errors + ansible.builtin.assert: + that: + - "{{ pre_check[check_id].input_error_count | int }} <= {{ post_check[check_id].input_error_count | int }}" + fail_msg: "WARNING: Interface input errors have been increased" + success_msg: "PASS: Interface input errors have not been increased " + failed_when: false + +- name: Compare PRE and POST interface output errors + ansible.builtin.assert: + that: + - "{{ pre_check[check_id].output_error_count | int }} <= {{ post_check[check_id].output_error_count | int }}" + fail_msg: "WARNING: Interface output errors have been increased" + success_msg: "PASS: Interface output errors have not been increased " + failed_when: false + +- name: Set facts + ansible.builtin.set_fact: + tx_power_dbm_post: "{{ post_check[check_id].output_power_dbm }}" + tx_power_dbm_pre: "{{ pre_check[check_id].output_power_dbm }}" + rx_power_dbm_post: "{{ post_check[check_id].rx_signal_avg_power_dbm }}" + rx_power_dbm_pre: "{{ pre_check[check_id].rx_signal_avg_power_dbm }}" + input_error_count_post: "{{ post_check[check_id].input_error_count }}" + input_error_count_pre: "{{ pre_check[check_id].input_error_count }}" + output_error_count_post: "{{ post_check[check_id].output_error_count }}" + output_error_count_pre: "{{ pre_check[check_id].output_error_count }}" + when: + - pre_check[check_id] is defined + - post_check[check_id] is defined + +- name: Set facts calculate diffs + ansible.builtin.set_fact: + tx_power_dbm_diff: "{{ '{:0.2f}'.format(tx_power_dbm_post | float - tx_power_dbm_pre | float) }}" + rx_power_dbm_diff: "{{ '{:0.2f}'.format(rx_power_dbm_post | float - rx_power_dbm_pre | float) }}" + input_error_count_diff: "{{ input_error_count_post | int - input_error_count_pre | int }}" + output_error_count_diff: "{{ output_error_count_post | int - output_error_count_pre | int }}" + +- name: Add PRE and POST comparison values to yaml + ansible.builtin.blockinfile: + path: "roles/check_demarcation_point/vars/{{ check_id }}.yml" + marker: "# {mark} -------------- DIFFS (POST - PRE) ----------------------------- #" + block: | + {{ verb + ':' }} + {{ check_id + ':' }} + {{ 'tx_power_diff:' }} {{ tx_power_dbm_diff }} + {{ 'rx_power_diff:' }} {{ rx_power_dbm_diff }} + {{ 'input_error_count_diff:' }} {{ input_error_count_diff }} + {{ 'output_error_count_diff:' }} {{ output_error_count_diff }} + when: + - pre_check[check_id] is defined + - post_check[check_id] is defined + - file_status.stat.exists + +- name: Check if Rx power in POST check is way lower than PRE check (above permitted threshold) + ansible.builtin.assert: + that: + - "{{ rx_power_dbm_diff | float }} > {{ rx_power_allowed_diff | float }} " + fail_msg: "WARNING: Rx Power (dBm) diff between pre/post check is bigger than allowed threshold." + success_msg: "PASS: Rx Power (dBm) diff between pre/post check is normal." + failed_when: false + +- name: Check if Tx power in POST check is way lower than PRE check (above permitted threshold) + ansible.builtin.assert: + that: + - "{{ tx_power_dbm_diff | float }} > {{ tx_power_allowed_diff | float }} " + fail_msg: "WARNING: Rx Power (dBm) diff between pre/post check is bigger than allowed threshold." + success_msg: "PASS: Rx Power (dBm) diff between pre/post check is normal." + failed_when: false + +- name: Check if input_errors increase diff between PRE and POST checks is above permitted threshold + ansible.builtin.assert: + that: + - "{{ input_error_count_diff | int }} < {{ input_error_count_threshold | int }} " + fail_msg: "WARNING: Input errors count in POST check have been increased above threshold." + success_msg: "PASS: Input errors count diffs between PRE and POST check is normal." + failed_when: false + +- name: Check if output_errors increase diff between PRE and POST checks is above permitted threshold + ansible.builtin.assert: + that: + - "{{ output_error_count_diff | int }} < {{ output_error_count_threshold | int }} " + fail_msg: "WARNING: Output errors count in POST check have been increased above threshold." + success_msg: "PASS: Output errors count diffs between PRE and POST check is normal." + failed_when: false \ No newline at end of file diff --git a/roles/check_demarcation_point/tasks/create_file.yaml b/roles/check_demarcation_point/tasks/create_file.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c4a1ba362e269e3ece00ff29fb8a2f9c01c9c1ed --- /dev/null +++ b/roles/check_demarcation_point/tasks/create_file.yaml @@ -0,0 +1,49 @@ +--- + +- name: Get output file information + ansible.builtin.stat: + path: roles/check_demarcation_point/vars/{{ check_id }}.yml + register: file_status + +- name: Creating a yaml file with pre_check results content + ansible.builtin.copy: + content: | + {{ '---' }} + {{ verb + ':' }} + {{ check_id + ':' }} + {{ 'interface:' }} {{ interface }} + {{ 'device:' }} {{ inventory_hostname }} + {{ 'interface_status:' }} {{ interface_status }} + {{ 'input_error_count:' }} {{ input_error_count }} + {{ 'output_error_count:' }} {{ output_error_count }} + {{ 'ethernet_pcs_statistics_bit_error_secs:' }} {{ ethernet_pcs_statistics_bit_error_secs }} + {{ 'ethernet_pcs_statistics_errored_blocks_secs:' }} {{ ethernet_pcs_statistics_errored_blocks_secs }} + {{ 'ethernet_fec_statistics:' }} {{ ethernet_fec_statistics }} + {{ 'output_power_dbm:' }} {{ output_power_dbm }} + {{ 'rx_signal_avg_power_dbm:' }} {{ rx_signal_avg_power_dbm }} + dest: "roles/check_demarcation_point/vars/{{ check_id }}.yml" + mode: '0660' + when: + - not file_status.stat.exists + + +- name: Add post_check parameters to yaml + ansible.builtin.blockinfile: + path: "roles/check_demarcation_point/vars/{{ check_id }}.yml" + marker: "# {mark} -------------- POST CHECKS ----------------------------- #" + block: | + {{ verb + ':' }} + {{ check_id + ':' }} + {{ 'interface:' }} {{ interface }} + {{ 'device:' }} {{ inventory_hostname }} + {{ 'interface_status:' }} {{ interface_status }} + {{ 'input_error_count:' }} {{ input_error_count }} + {{ 'output_error_count:' }} {{ output_error_count }} + {{ 'ethernet_pcs_statistics_bit_error_secs:' }} {{ ethernet_pcs_statistics_bit_error_secs }} + {{ 'ethernet_pcs_statistics_errored_blocks_secs:' }} {{ ethernet_pcs_statistics_errored_blocks_secs }} + {{ 'ethernet_fec_statistics:' }} {{ ethernet_fec_statistics }} + {{ 'output_power_dbm:' }} {{ output_power_dbm }} + {{ 'rx_signal_avg_power_dbm:' }} {{ rx_signal_avg_power_dbm }} + when: + - verb == 'post_check' + - file_status.stat.exists diff --git a/roles/check_demarcation_point/tasks/get_ifstatus.yaml b/roles/check_demarcation_point/tasks/get_ifstatus.yaml new file mode 100644 index 0000000000000000000000000000000000000000..642bebf32e8b99b5d4066dbf209d6401725dad85 --- /dev/null +++ b/roles/check_demarcation_point/tasks/get_ifstatus.yaml @@ -0,0 +1,59 @@ +--- + +############################################################################################################### +# The playbook does the following: +# +# - Runs the command show 'interface statistics {{ interface }}' and gets the json output +# - From json output it retrieves the apporpriate metrics (status,input/output errors, FEC errors, PCS errors) +# - Saves the metrics in relevant variable names. +# +############################################################################################################### + +- name: Print input from user (DEBUG) + ansible.builtin.debug: + msg: + - "type of check is: {{ verb }}" + - "interface input from user is: {{ interface }}" + - "Vendor input from user is: {{ vendor }}" + - "check_id input from user is: {{ check_id }}" + +- name: Get Junos attributes for interface {{ interface }} + junipernetworks.junos.junos_command: + commands: "show interface statistics {{ interface }}" + display: json + register: interface_status + when: vendor == 'juniper' + + +- name: Manipulate the interface attributes + ansible.builtin.set_fact: + interface_status: "{{ interface_status.stdout_lines.0['interface-information'].0['physical-interface'].0['oper-status'].0['data']| default(False) }}" + input_error_count: "{{ interface_status.stdout_lines.0['interface-information'].0['physical-interface'].0['input-error-count'].0['data']| default(False) }}" + output_error_count: >- + {{ + interface_status.stdout_lines.0['interface-information'] + .0['physical-interface'].0['output-error-count'].0['data'] | default(False) + }} + ethernet_pcs_statistics_bit_error_secs: >- + "{{ interface_status.stdout_lines.0['interface-information'].0['physical-interface'] + .0['ethernet-pcs-statistics'].0['bit-error-seconds'].0['data']| default(False) + }}" + ethernet_pcs_statistics_errored_blocks_secs: >- + "{{ interface_status.stdout_lines.0['interface-information'].0['physical-interface'] + .0['ethernet-pcs-statistics'].0['errored-blocks-seconds'].0['data']| default(False) }}" + ethernet_fec_statistics: "{{ interface_status.stdout_lines.0['interface-information'].0['physical-interface'].0['ethernet-fec-statistics'].0 + | default(False) }}" + when: vendor == 'juniper' + + +- name: Print interface statistics + ansible.builtin.debug: + var: print_variables + vars: + print_variables: + interface_status: "{{ interface_status }}" + input_error_count: "{{ input_error_count }}" + output_error_count: "{{ output_error_count }}" + ethernet_pcs_statistics_bit_error_secs: "{{ ethernet_pcs_statistics_bit_error_secs }}" + ethernet_pcs_statistics_errored_blocks_secs: "{{ ethernet_pcs_statistics_errored_blocks_secs }}" + ethernet_fec_statistics: "{{ ethernet_fec_statistics }}" diff --git a/roles/check_demarcation_point/tasks/get_optical_levels.yaml b/roles/check_demarcation_point/tasks/get_optical_levels.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a1ffd75ef3fd4ad6c9c0cc6dfb2d32ed75d1a3e4 --- /dev/null +++ b/roles/check_demarcation_point/tasks/get_optical_levels.yaml @@ -0,0 +1,40 @@ +--- +############################################################################################################### +# The playbook does the following: +# +# +# - Runs the command 'show interfaces diagnostics optics {{ interface }} | no-more' and gets the json output +# - From json output it retrieves the apporpriate metrics for optical levels +# - Saves the metrics in relevant variable names +############################################################################################################### + +- name: Get Junos attributes for optical levels of interface {{ interface }} + junipernetworks.junos.junos_command: + commands: "show interfaces diagnostics optics {{ interface }} | no-more" + display: json + register: optical_status + when: vendor == 'juniper' + +- name: Manipulate the optical results attributes + ansible.builtin.set_fact: + output_power_dbm: >- + {{ + optical_status.stdout_lines.0['interface-information'] + .0['physical-interface'].0['optics-diagnostics'] + .0['laser-output-power-dbm'].0['data'] | default(False) + }} + rx_signal_avg_power_dbm: >- + {{ + optical_status.stdout_lines.0['interface-information'] + .0['physical-interface'].0['optics-diagnostics'] + .0['rx-signal-avg-optical-power-dbm'].0['data'] | default(False) + }} + when: vendor == 'juniper' + +- name: Print optical levels statistics + ansible.builtin.debug: + var: print_variables + vars: + print_variables: + output_power_dbm: "{{ output_power_dbm }}" + rx_signal_avg_power_dbm: "{{ rx_signal_avg_power_dbm }}" diff --git a/roles/check_demarcation_point/tasks/main.yml b/roles/check_demarcation_point/tasks/main.yml new file mode 100644 index 0000000000000000000000000000000000000000..42e84db221433d312aeeb49363e66206f5aea8a7 --- /dev/null +++ b/roles/check_demarcation_point/tasks/main.yml @@ -0,0 +1,41 @@ +--- +- name: Print the usage + ansible.builtin.debug: + msg: + - "Required arguments: 'verb', 'interface', 'vendor' and 'check_id' are required." + - "Allowed verbs: 'pre_check', 'post_check', 'compare'. Use: -e 'verb=$verb'" + - "Allowed interface: user_input Use: -e 'interface=$interface'" + - "Allowed vendors: 'juniper', 'cisco'. Use: -e 'vendor=$vendor'" + - "Allowed check_id: user_input Use: -e 'check_id=$check_id'" + when: + - (verb is not defined) or (interface is not defined) or (vendor is not defined) or (check_id is not defined) + +- name: End playbook + ansible.builtin.meta: end_play + when: + - (verb is not defined) or (interface is not defined) or (vendor is not defined) or (check_id is not defined) + +- name: Include task get_ifstatus + ansible.builtin.include_tasks: get_ifstatus.yaml + when: + - verb in ['pre_check','post_check'] + +- name: Include task get_optical_levels + ansible.builtin.include_tasks: get_optical_levels.yaml + when: + - verb in ['pre_check','post_check'] + +- name: Include task create_file + ansible.builtin.include_tasks: create_file.yaml + when: + - verb in ['pre_check','post_check'] + +- name: Include task checks + ansible.builtin.include_tasks: checks.yaml + when: + - verb in ['pre_check','post_check'] + +- name: Include task compare + ansible.builtin.include_tasks: compare.yaml + when: + - verb == 'compare' diff --git a/roles/check_demarcation_point/vars/main.yml b/roles/check_demarcation_point/vars/main.yml new file mode 100644 index 0000000000000000000000000000000000000000..4707c560807cb1ab26e26a5942c7e083a8da127f --- /dev/null +++ b/roles/check_demarcation_point/vars/main.yml @@ -0,0 +1,17 @@ +--- +# vars file for demarcation_point checks +dryrun: "True" +verbs: + - "pre_check" + - "post_check" + - "compare" +vendors: + - "juniper" + - "cisco" +# config_objects: +# - "trunk_interface" +# - "isis" +# - "isis_interface" +# - "ldp_interface" +# - "lldp_interface" +# - "trunk_deprovision" diff --git a/roles/check_demarcation_point/vars/optical_thresholds.yml b/roles/check_demarcation_point/vars/optical_thresholds.yml new file mode 100644 index 0000000000000000000000000000000000000000..0c81c198ad197e44730035c528c2e4dafcd2bf1a --- /dev/null +++ b/roles/check_demarcation_point/vars/optical_thresholds.yml @@ -0,0 +1,15 @@ +--- +min_tx_power_dbm: -8 +max_tx_power_dbm: 0 +min_rx_power_dbm: -10 +max_rx_power_dbm: -0.5 +rx_power_allowed_diff: -2 +tx_power_allowed_diff: -2 +input_error_count_threshold: 3 +output_error_count_threshold: 3 +fec_ccw_count_threshold: 0 +fec_ccw_error_rate_threshold: 0 +fec_nccw_count_threshold: 0 +fec_nccw_error_rate_threshold: 0 +pcs_bit_error_seconds_threshold: 20 +pcs_bit_error_blocks_seconds_threshold: 20 \ No newline at end of file