diff --git a/flowspy/settings.py.dist b/flowspy/settings.py.dist index 2b90aeb73d2b9e414a913a9a34e6919c5102155e..850509aa9f49d0169281e6fbbb2e789a2bbd704d 100644 --- a/flowspy/settings.py.dist +++ b/flowspy/settings.py.dist @@ -567,6 +567,8 @@ NETCONF_PORT = 830 # generic mitigation statistics support MITIGATION_STATISTIC_COLLECTOR_SPECIFIC_CLASS="mitigation_stats_collector_specific_junos_snmp" +#MITIGATION_STATISTIC_COLLECTOR_SPECIFIC_CLASS="mitigation_stats_collector_specific_nokia" +#MITIGATION_STATISTIC_COLLECTOR_SPECIFIC_CLASS="mitigation_stats_collector_specific_multi" ############################################################################## ############################################################################## diff --git a/utils/dfncert/fodadapter.py b/utils/dfncert/fodadapter.py index 2c876f3854c2cede672edcf3ae1cdca36e14b3a5..be3abf84cb3574ca3f73cf2b7cfb4466615ff275 100644 --- a/utils/dfncert/fodadapter.py +++ b/utils/dfncert/fodadapter.py @@ -130,10 +130,13 @@ def get_nokia_stats(): logger.info("get_nokia_stats(): before creating manager") #multiprocessing.set_start_method('spawn') + + ## hack to make current celery worker process non-daemonic, so it can reuse multiprocessing: + #del (multiprocessing.current_process()._config)['daemon'] #manager = multiprocessing.Manager() import queue - #q = queue.Queue() + ##q = queue.Queue() manager = queue logger.info("get_nokia_stats(): after creating manager") @@ -145,7 +148,7 @@ def get_nokia_stats(): if option_oneshot: - logger.info("get_nokia_stats(): option_oneshot") + logger.info("get_nokia_stats(): option_oneshotx ROUTERS="+str(ROUTERS)) router_data = {router['name']: query_router_once(router) for router in ROUTERS} diff --git a/utils/mitigation_stats_collector.py b/utils/mitigation_stats_collector.py index dfad25659ebf62110099d49be9cd67ca5e212ec2..cce3e1957a389984a5e08afd72477dce3fd2cdd4 100644 --- a/utils/mitigation_stats_collector.py +++ b/utils/mitigation_stats_collector.py @@ -11,6 +11,9 @@ if hasattr(settings, "MITIGATION_STATISTIC_COLLECTOR_SPECIFIC_CLASS") and settin elif hasattr(settings, "MITIGATION_STATISTIC_COLLECTOR_SPECIFIC_CLASS") and settings.MITIGATION_STATISTIC_COLLECTOR_SPECIFIC_CLASS == "mitigation_stats_collector_specific_junos_snmp": from utils import mitigation_stats_collector_specific_junos_snmp as mitigation_stats_collector_specific_class mitigation_stats_collector_specific = mitigation_stats_collector_specific_class.MitigationStatisticCollectorSpecific_JunosSnmp() +elif hasattr(settings, "MITIGATION_STATISTIC_COLLECTOR_SPECIFIC_CLASS") and settings.MITIGATION_STATISTIC_COLLECTOR_SPECIFIC_CLASS == "mitigation_stats_collector_specific_multi": + from utils import mitigation_stats_collector_specific_multi as mitigation_stats_collector_specific_class + mitigation_stats_collector_specific = mitigation_stats_collector_specific_class.MitigationStatisticCollectorSpecific_Multi() else: # default is junos_snmp for compatibility from utils import mitigation_stats_collector_specific_junos_snmp as mitigation_stats_collector_specific_class mitigation_stats_collector_specific = mitigation_stats_collector_specific_class.MitigationStatisticCollectorSpecific_JunosSnmp() diff --git a/utils/mitigation_stats_collector_specific_multi.py b/utils/mitigation_stats_collector_specific_multi.py new file mode 100644 index 0000000000000000000000000000000000000000..b6d04b6e965484abedbbec52bc7edb3bb4c90b1e --- /dev/null +++ b/utils/mitigation_stats_collector_specific_multi.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- vim:fileencoding=utf-8: +# vim: tabstop=4:shiftwidth=4:softtabstop=4:expandtab + +from pysnmp.hlapi.asyncore import * +from django.conf import settings +from datetime import datetime, timedelta +import json +import os +import time +import re + +from flowspec.models import Route +from utils.route_spec_utils import get_rulename_by_ruleparams__generic, unify_ratelimit_value + +from utils.mitigation_stats_collector_specific_base import MitigationStatisticCollectorSpecific_Base +from utils.mitigation_stats_collector_specific_junos_snmp import MitigationStatisticCollectorSpecific_JunosSnmp +from utils.mitigation_stats_collector_specific_nokia import MitigationStatisticCollectorSpecific_Nokia + +import flowspec.logging_utils +logger = flowspec.logging_utils.logger_init_default(__name__, "celery_multistats.log", False) + +# + +class MitigationStatisticCollectorSpecific_Multi(MitigationStatisticCollectorSpecific_Base): + + # + + # to be overriden in sub classes + def get_new_mitigation_statistic_data(self): + return self.get_statistic_data__from_multiple_sources() + + # to be overriden in sub classes + def get_statistic_data_rule_key(self, ruleobj): + return get_rulename_by_ruleparams__generic(ruleobj) + + + def get_statistic_data__from_multiple_sources(self): + + # TODO: run all parallel: + + mitigation_stats_collector_specific1 = MitigationStatisticCollectorSpecific_Nokia() + stats_data1 = mitigation_stats_collector_specific1.get_new_mitigation_statistic_data() + + + mitigation_stats_collector_specific2 = MitigationStatisticCollectorSpecific_JunosSnmp() + stats_data2 = mitigation_stats_collector_specific2.get_new_mitigation_statistic_data() + + # TODO: merge data +