diff --git a/doc/development/format_of_snmp_json_db_file.txt b/doc/development/format_of_snmp_json_db_file.txt
new file mode 100644
index 0000000000000000000000000000000000000000..e5387ea48ffb910dbef04ab37f686009f5613080
--- /dev/null
+++ b/doc/development/format_of_snmp_json_db_file.txt
@@ -0,0 +1,53 @@
+
+statistics table:
+
+a measurement value
+per rule,
+for each measurement time slot,
+
+e.g.,
+
+{
+ "_per_rule": {
+ "243": [
+ {"ts": "2023-09-28T13:00:02.107721", "value": {"packets": 0, "bytes": 0 },
+ {"ts": "2023-09-28T13:00:07.107721", "value": {"packets": 10, "bytes": 4545 } },
+ {"ts": "2023-09-28T13:00:12.107721", "value": 1,
+ {"ts": "2023-09-28T13:00:17.107721", "value": {"packets": 30, "bytes": 8539 } },
+ {"ts": "2023-09-28T13:00:22.107721", "value": {"packets": 80, "bytes": 18234 } },
+ {"ts": "2023-09-28T13:00:27.107721", "value": {"packets": 80, "bytes": 18234 } },
+ {"ts": "2023-09-28T13:00:32.107721", "value": {"packets": 80, "bytes": 18234 } },
+ {"ts": "2023-09-28T13:00:33.107721", "value": 0 },
+ {"ts": "2023-09-30T14:00:11.107721", "value": {"packets": 0, "bytes": 0 },
+ {"ts": "2023-09-30T14:00:12.107721", "value": {"packets": 15, "bytes": 3691 } },
+ {"ts": "2023-09-30T14:00:17.107721", "value": {"packets": 25, "bytes": 18539 } },
+ {"ts": "2023-09-30T14:00:22.107721", "value": {"packets": 120, "bytes": 28934 } },
+ {"ts": "2023-09-30T14:00:27.107721", "value": {"packets": 120, "bytes": 28934 } },
+ {"ts": "2023-09-30T14:00:32.107721", "value": {"packets": 120, "bytes": 28934 } },
+ {"ts": "2023-09-30T14:00:37.107721", "value": {"packets": 120, "bytes": 28934 } },
+ {"ts": "2023-09-30T14:00:42.107721", "value": {"packets": 120, "bytes": 28934 } },
+ {"ts": "2023-09-30T14:00:47.107721", "value": {"packets": 120, "bytes": 28934 } },
+ {"ts": "2023-09-28T14:00:49.107721", "value": 0 },
+ ],
+ ...
+ },
+}
+
+measurement value per time slot is either:
+
+zero measurement: = { bytes=>0, packets=>0, } : at start of measurement = activation of the rule on the router
+
+or
+
+real measurement (example): = { bytes=>1414, packets=>5, } : cumulative (=absolue values from start of measurement = from last activation of the rule) bytes/packet counters
+
+or
+
+missing_value: =1 : no measurements from none of the routers queried answered values for the specific rule in the specific scheduling time slot (while rule is activated)
+
+null_value: =0 : used when rule is deactivated = measurement should stop on the router
+
+when rule is re-activated again on the router after some duration of having been deactivated it is started again with a zero measurement in the table and absolute values for the future real measurement are starting from zero again
+
+
+
diff --git a/flowspec/snmpstats.py b/flowspec/snmpstats.py
index b47a831150c2c18c4f78e23f5c7743d831803168..5cda2cb8c4944d8fe91bcf350436fd688cecad7c 100644
--- a/flowspec/snmpstats.py
+++ b/flowspec/snmpstats.py
@@ -23,6 +23,7 @@ from datetime import datetime, timedelta
import json
import os
import time
+import re
from flowspec.models import Route
from flowspec.junos import create_junos_name
@@ -36,17 +37,19 @@ identoffset = len(settings.SNMP_CNTPACKETS) + 1
# noinspection PyUnusedLocal,PyUnusedLocal
def snmpCallback(snmpEngine, sendRequestHandle, errorIndication,
errorStatus, errorIndex, varBindTable, cbCtx):
+ try:
(authData, transportTarget, results) = cbCtx
+ logger.error('snmpCallback(): called')
# debug - which router replies:
#print('%s via %s' % (authData, transportTarget))
# CNTPACKETS and CNTBYTES are of the same length
if errorIndication:
- logger.error('Bad errorIndication.')
+ logger.error('snmpCallback(): Bad errorIndication.')
return 0
elif errorStatus:
- logger.error('Bad errorStatus.')
+ logger.error('snmpCallback(): Bad errorStatus.')
return 0
for varBindRow in varBindTable:
for name, val in varBindRow:
@@ -56,7 +59,7 @@ def snmpCallback(snmpEngine, sendRequestHandle, errorIndication,
elif name.startswith(settings.SNMP_CNTBYTES):
counter = "bytes"
else:
- logger.debug('Finished {}.'.format(transportTarget))
+ logger.debug('snmpCallback(): Finished {}.'.format(transportTarget))
return 0
ident = name[identoffset:]
@@ -70,28 +73,38 @@ def snmpCallback(snmpEngine, sendRequestHandle, errorIndication,
len2 = ordvals[len1] + 1
routename = "".join([chr(i) for i in ordvals[len1 + 1:len1 + len2]])
+ logger.error("routename="+str(routename))
+ xtype='counter'
+ if re.match(r'^[0-9]+[Mk]_', routename):
+ ary=re.split(r'_', routename, maxsplit=1)
+ xtype=ary[0]
+ routename=ary[1]
+ logger.error("=> routename="+str(routename)+" xtype="+str(xtype))
+
# add value into dict
if routename in results:
- if counter in results[routename]:
- results[routename][counter] = results[routename][counter] + int(val)
+ if counter in results[routename][xtype]:
+ results[routename][xtype][counter] = results[routename][counter][xtype] + int(val)
else:
- results[routename][counter] = int(val)
+ results[routename][xtype][counter] = int(val)
else:
- results[routename] = {counter: int(val)}
+ results[routename] = { xtype: {counter: int(val)} }
logger.debug("%s %s %s %s = %s" %(transportTarget, counter, tablename, routename, int(val)))
-
- return 1 # continue table retrieval
+
+ except Exception as e:
+ logger.error("snmpCallback(): got exception "+str(e), exc_info=True)
+ return 1 # continue table retrieval
def get_snmp_stats():
- """Return dict() of the sum of counters (bytes, packets) from all selected routes, where
- route identifier is the key in dict. The sum is counted over all routers.
-
- Example output with one rule: {'77.72.72.1,0/0,proto=1': {'bytes': 13892216, 'packets': 165387}}
+ """Return dict() of the sum of counters (bytes, packets) from all selected routes, where
+ route identifier is the key in dict. The sum is counted over all routers.
- This function uses SNMP_IP list, SNMP_COMMUNITY, SNMP_CNTPACKETS and
- SNMP_RULESFILTER list, all defined in settings."""
+ Example output with one rule: {'77.72.72.1,0/0,proto=1': {'bytes': 13892216, 'packets': 165387}}
+ This function uses SNMP_IP list, SNMP_COMMUNITY, SNMP_CNTPACKETS and
+ SNMP_RULESFILTER list, all defined in settings."""
+ try:
if not isinstance(settings.SNMP_IP, list):
settings.SNMP_IP = [settings.SNMP_IP]
@@ -132,6 +145,8 @@ def get_snmp_stats():
snmpEngine.transportDispatcher.runDispatcher()
return results
+ except Exception as e:
+ logger.error("get_snmp_stats(): got exception "+str(e), exc_info=True)
def lock_history_file(wait=1, reason=""):
first=1
@@ -272,12 +287,17 @@ def poll_snmp_statistics():
# proper update history
samplecount = settings.SNMP_MAX_SAMPLECOUNT
for rule in newdata:
- counter = {"ts": nowstr, "value": newdata[rule]}
- if rule in history:
- history[rule].insert(0, counter)
- history[rule] = history[rule][:samplecount]
- else:
- history[rule] = [counter]
+ for xtype in newdata[rule]:
+ key = "value"
+ if xtype!="counter":
+ key = "value-"+str(xtype)
+ #counter = {"ts": nowstr, "value": newdata[rule]['counter']}
+ counter = {"ts": nowstr, key: newdata[rule][xtype]}
+ if rule in history:
+ history[rule].insert(0, counter)
+ history[rule] = history[rule][:samplecount]
+ else:
+ history[rule] = [counter]
# check for old rules and remove them
toremove = []
@@ -310,7 +330,15 @@ def poll_snmp_statistics():
for ruleobj in queryset:
rule_id = str(ruleobj.id)
rule_status = str(ruleobj.status).upper()
- logger.debug("snmpstats: STATISTICS_PER_RULE rule_id="+str(rule_id)+" rule_status="+str(rule_status))
+
+ xtype='counter'
+ limit_rate = None
+ for thenaction in ruleobj.then.all():
+ if thenaction.action and thenaction.action=='rate-limit':
+ limit_rate=thenaction.action_value
+ xtype=str(limit_rate)
+
+ logger.debug("snmpstats: STATISTICS_PER_RULE rule_id="+str(rule_id)+" rule_status="+str(rule_status)+" limit_rate="+str(limit_rate))
#rule_last_updated = str(ruleobj.last_updated) # e.g. 2018-06-21 08:03:21+00:00
#rule_last_updated = datetime.strptime(str(ruleobj.last_updated), '%Y-%m-%d %H:%M:%S+00:00') # TODO TZ offset assumed to be 00:00
rule_last_updated = helper_rule_ts_parse(str(ruleobj.last_updated))
@@ -326,7 +354,7 @@ def poll_snmp_statistics():
if rule_status=="ACTIVE":
try:
- counter = {"ts": nowstr, "value": newdata[flowspec_params_str]}
+ counter = {"ts": nowstr, "value": newdata[flowspec_params_str][xtype]}
counter_is_null = False
except Exception as e:
logger.info("poll_snmp_statistics(): 1 STATISTICS_PER_RULE: exception: rule_id="+str(rule_id)+" newdata for flowspec_params_str='"+str(flowspec_params_str)+"' missing : "+str(e))
diff --git a/flowspec/tasks.py b/flowspec/tasks.py
index e2ce3f72601cd38772f436c3c1293aa38ff11709..d503e907308b3fa0d70d65562b2e24b5d4c091c1 100644
--- a/flowspec/tasks.py
+++ b/flowspec/tasks.py
@@ -76,6 +76,8 @@ def edit(routepk, callback=None):
commit, response = applier.apply(operation="replace")
if commit:
route.status = "ACTIVE"
+ route.response = response
+ route.save() # save() has to be called before snmp_add_initial_zero_value, as last_updated DB filed is updated now on every call of save() and last db_measurement time must become >= this new last_updated value
try:
#snmp_add_initial_zero_value.delay(str(route.id), True)
snmp_add_initial_zero_value(str(route.id), True)
@@ -86,10 +88,14 @@ def edit(routepk, callback=None):
# repeat the action
raise TimeoutError()
route.status = "ERROR"
+ route.response = response
+ route.save()
else:
route.status = "ERROR"
- route.response = response
- route.save()
+ route.response = response
+ route.save()
+ #route.response = response
+ #route.save()
announce("[%s] Rule edit: %s - Result: %s" % (route.applier_username_nice, route.name_visible, response), route.applier, route)
@shared_task(ignore_result=True, autoretry_for=(TimeoutError, TimeLimitExceeded, SoftTimeLimitExceeded), retry_backoff=True, retry_kwargs={'max_retries': settings.NETCONF_MAX_RETRY_BEFORE_ERROR})
diff --git a/vnet_router/snmp/pass_persisttest_bgpflowspec b/vnet_router/snmp/pass_persisttest_bgpflowspec
index d95210cb08f1cc7e63177d4846343a356e3c38d0..7adde9e8bb4add2b1fccd7e31a8599c359c5513a 100755
--- a/vnet_router/snmp/pass_persisttest_bgpflowspec
+++ b/vnet_router/snmp/pass_persisttest_bgpflowspec
@@ -105,6 +105,10 @@ sub get_snmp_rulename
$src =~ s/^0.0.0.0\/0$/0\/0/;
$dst =~ s/^0.0.0.0\/0$/0\/0/;
+ # https://github.com/librenms/librenms/blob/master/mibs/junos/JUNIPER-FIREWALL-MIB
+
+ my $counter_type_value = 2; # default is normal counter
+
my $rulename = $dst.",".$src;
my $proto = $protocol_num;
@@ -144,9 +148,17 @@ sub get_snmp_rulename
$rulename .= ",frag".join(",", (map { $frag_types->{$_}; } split(/,/, $fragment_options)));
}
+ if (defined($thenaction)) {
+ if ($thenaction eq 'discard') {
+ } elsif ($thenaction =~ /^rate-limit-([0-9]+([Mk]?))$/) {
+ $rulename = $1."_".$rulename;
+ $counter_type_value = 3; # policer
+ }
+ }
+
print STDERR "rulename=$rulename$/" if ($debug);
- return $rulename;
+ return ($rulename, $counter_type_value);
}
sub get_state($)
@@ -163,11 +175,11 @@ sub get_state($)
my ($cookie, $typex, $byte_ctr, $pkt_ctr, $rulename, @rule_params) = split(/\s+/, $line);
next if ($typex eq 'default');
- my $snmp_rulename = get_snmp_rulename(@rule_params);
+ my ($snmp_rulename, $counter_type_value) = get_snmp_rulename(@rule_params);
if ($snmp_rulename ne '') {
my $prefix1 = string_to_oid_list($snmp_rulename);
- my $snmp_oid_counter_pkts = $place_pkts.$rulesfilter_oidlist.$prefix1.".2";
- my $snmp_oid_counter_bytes = $place_bytes.$rulesfilter_oidlist.$prefix1.".2";
+ my $snmp_oid_counter_pkts = $place_pkts.$rulesfilter_oidlist.$prefix1.".".$counter_type_value;
+ my $snmp_oid_counter_bytes = $place_bytes.$rulesfilter_oidlist.$prefix1.".".$counter_type_value;
$state_hash->{$snmp_oid_counter_bytes} = $byte_ctr;
$state_hash->{$snmp_oid_counter_pkts} = $pkt_ctr;
}