Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
M
Mapping Provider
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Wiki
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Package registry
Container registry
Model registry
Operate
Environments
Terraform modules
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
geant-swd
Next Generation Map
Mapping Provider
Commits
725960bc
Commit
725960bc
authored
1 month ago
by
Erik Reid
Browse files
Options
Downloads
Patches
Plain Diff
added/update schemas on reporting/inventory requests & cache
parent
2d97a5f6
No related branches found
No related tags found
No related merge requests found
Changes
2
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
mapping_provider/backends/inventory.py
+106
-11
106 additions, 11 deletions
mapping_provider/backends/inventory.py
test/test_inventory.py
+11
-11
11 additions, 11 deletions
test/test_inventory.py
with
117 additions
and
22 deletions
mapping_provider/backends/inventory.py
+
106
−
11
View file @
725960bc
import
concurrent.futures
import
concurrent.futures
import
jsonschema
import
logging
import
logging
import
time
import
time
from
threading
import
Event
from
threading
import
Event
...
@@ -10,15 +11,104 @@ from . import cache
...
@@ -10,15 +11,104 @@ from . import cache
logger
=
logging
.
getLogger
(
__name__
)
logger
=
logging
.
getLogger
(
__name__
)
INPROV_POLLER_INTERFACES_CACHE_FILENAME
=
'
inprov-poller-interfaces.json
'
#
INPROV_POLLER_INTERFACES_CACHE_FILENAME = 'inprov-poller-interfaces.json'
REPORTING_SCID_CURRENT_CACHE_FILENAME
=
'
reporting-scid-current.json
'
REPORTING_SCID_CURRENT_CACHE_FILENAME
=
'
reporting-scid-current.json
'
INPROV_MAP_SERVICES_CACHE_FILENAME
=
'
inprov-map-services.json
'
# INPROV_MAP_SERVICES_CACHE_FILENAME = 'inprov-map-services.json'
INPROV_EQUIPMENT_CACHE_FILENAME
=
'
inprov-equipment.json
'
REPORTING_SCID_CURRENT_CACHE_SCHEMA
=
{
'
$schema
'
:
'
https://json-schema.org/draft/2020-12/schema
'
,
'
definitions
'
:
{
'
interface
'
:
{
'
type
'
:
'
object
'
,
'
properties
'
:
{
'
hostname
'
:
{
'
type
'
:
'
string
'
},
# 'interface': {'type': 'string'},
# 'addresses': {
# 'type': 'array',
# 'items': {'type': 'string'}
# }
},
'
required
'
:
[
'
hostname
'
]
# 'required': ['hostname', 'interface']
},
'
lambda_interface
'
:
{
'
type
'
:
'
object
'
,
'
properties
'
:
{
'
equipment
'
:
{
'
type
'
:
'
string
'
},
# 'port': {'type': 'string'},
},
'
required
'
:
[
'
equipment
'
]
# 'required': ['equipment', 'port']
},
'
service
'
:
{
'
type
'
:
'
object
'
,
'
properties
'
:
{
'
scid
'
:
{
'
type
'
:
'
string
'
},
'
sid
'
:
{
'
type
'
:
'
string
'
},
'
name
'
:
{
'
type
'
:
'
string
'
},
'
speed
'
:
{
'
type
'
:
'
integer
'
},
'
status
'
:
{
'
type
'
:
'
string
'
},
# 'monitored': {'type': 'boolean'},
'
service_type
'
:
{
'
type
'
:
[
'
string
'
,
'
null
'
]},
# 'imsid': {'type': 'integer'},
# 'customers': {
# 'type': 'array',
# 'items': {'type': 'string'}
# },
'
endpoints
'
:
{
'
type
'
:
'
array
'
,
'
items
'
:
{
'
anyOf
'
:
[
{
'
$ref
'
:
'
#/definitions/interface
'
},
{
'
$ref
'
:
'
#/definitions/lambda_interface
'
},
]
}
}
},
'
required
'
:
[
'
scid
'
,
'
sid
'
,
'
name
'
,
'
speed
'
,
'
status
'
,
'
service_type
'
,
'
endpoints
'
],
# 'required': ['scid', 'sid', 'name',
# 'speed', 'status', 'monitored',
# 'service_type', 'imsid', 'customers', 'endpoints'],
# 'additionalProperties': False
},
},
'
type
'
:
'
array
'
,
'
items
'
:
{
'
$ref
'
:
'
#/definitions/service
'
}
}
INPROV_EQUIPMENT_LIST_SCHEMA
=
{
'
$schema
'
:
'
https://json-schema.org/draft/2020-12/schema
'
,
'
definitions
'
:
{
'
equipment
'
:
{
'
type
'
:
'
object
'
,
'
properties
'
:
{
'
name
'
:
{
'
type
'
:
'
string
'
},
'
pop
'
:
{
'
type
'
:
'
string
'
},
'
status
'
:
{
'
type
'
:
'
string
'
},
},
'
required
'
:
[
'
name
'
,
'
pop
'
,
'
status
'
],
# 'additionalProperties': False
},
},
'
type
'
:
'
array
'
,
'
items
'
:
{
'
$ref
'
:
'
#/definitions/equipment
'
}
}
def
_load_and_cache_json
(
def
_load_and_cache_json
(
key
:
str
,
key
:
str
,
url
:
str
,
url
:
str
,
cache_filename
:
str
)
->
dict
[
str
,
Any
]:
cache_filename
:
str
,
schema
:
dict
[
str
,
Any
]
|
None
=
None
)
->
dict
[
str
,
Any
]:
"""
"""
Load the JSON from the URL, return and cache it.
Load the JSON from the URL, return and cache it.
...
@@ -32,6 +122,9 @@ def _load_and_cache_json(
...
@@ -32,6 +122,9 @@ def _load_and_cache_json(
rv
.
raise_for_status
()
rv
.
raise_for_status
()
rsp_object
=
rv
.
json
()
rsp_object
=
rv
.
json
()
if
schema
:
jsonschema
.
validate
(
instance
=
rsp_object
,
schema
=
schema
)
cache
.
set
(
cache_filename
,
rsp_object
)
cache
.
set
(
cache_filename
,
rsp_object
)
return
{
return
{
'
key
'
:
key
,
'
key
'
:
key
,
...
@@ -50,21 +143,23 @@ def _load_all_inventory(inventory_base_uri: str, reporting_base_uri: str) -> dic
...
@@ -50,21 +143,23 @@ def _load_all_inventory(inventory_base_uri: str, reporting_base_uri: str) -> dic
with
concurrent
.
futures
.
ThreadPoolExecutor
(
max_workers
=
1
)
as
executor
:
with
concurrent
.
futures
.
ThreadPoolExecutor
(
max_workers
=
1
)
as
executor
:
futures
=
[
futures
=
[
executor
.
submit
(
#
executor.submit(
_load_and_cache_json
,
#
_load_and_cache_json,
key
=
'
poller-interfaces
'
,
#
key='poller-interfaces',
url
=
f
'
{
inventory_base_uri
}
/poller/interfaces
'
,
#
url=f'{inventory_base_uri}/poller/interfaces',
cache_filename
=
INPROV_POLLER_INTERFACES_CACHE_FILENAME
),
#
cache_filename=INPROV_POLLER_INTERFACES_CACHE_FILENAME),
executor
.
submit
(
executor
.
submit
(
_load_and_cache_json
,
_load_and_cache_json
,
key
=
'
map-services
'
,
key
=
'
map-services
'
,
url
=
f
'
{
inventory_base_uri
}
/map/services
'
,
url
=
f
'
{
inventory_base_uri
}
/map/equipment
'
,
cache_filename
=
INPROV_MAP_SERVICES_CACHE_FILENAME
),
cache_filename
=
INPROV_EQUIPMENT_CACHE_FILENAME
,
schema
=
INPROV_EQUIPMENT_LIST_SCHEMA
),
executor
.
submit
(
executor
.
submit
(
_load_and_cache_json
,
_load_and_cache_json
,
key
=
'
scid-current
'
,
key
=
'
scid-current
'
,
url
=
f
'
{
reporting_base_uri
}
/scid/current
'
,
url
=
f
'
{
reporting_base_uri
}
/scid/current
'
,
cache_filename
=
REPORTING_SCID_CURRENT_CACHE_FILENAME
),
cache_filename
=
REPORTING_SCID_CURRENT_CACHE_FILENAME
,
schema
=
REPORTING_SCID_CURRENT_CACHE_SCHEMA
),
]
]
responses
=
{}
responses
=
{}
for
_f
in
concurrent
.
futures
.
as_completed
(
futures
):
for
_f
in
concurrent
.
futures
.
as_completed
(
futures
):
...
...
This diff is collapsed.
Click to expand it.
test/test_inventory.py
+
11
−
11
View file @
725960bc
...
@@ -16,11 +16,11 @@ def test_inventory_service_download():
...
@@ -16,11 +16,11 @@ def test_inventory_service_download():
inventory_base_uri
=
'
https://dummy-hostname.dummy.domain
'
inventory_base_uri
=
'
https://dummy-hostname.dummy.domain
'
reporting_base_uri
=
'
https://another-dummy-hostname.dummy.domain
'
reporting_base_uri
=
'
https://another-dummy-hostname.dummy.domain
'
responses
.
add
(
#
responses.add(
method
=
responses
.
GET
,
#
method=responses.GET,
url
=
f
'
{
inventory_base_uri
}
/poller/interfaces
'
,
#
url=f'{inventory_base_uri}/poller/interfaces',
json
=
load_test_data
(
'
poller-interfaces.json
'
)
#
json=load_test_data('poller-interfaces.json')
)
#
)
responses
.
add
(
responses
.
add
(
method
=
responses
.
GET
,
method
=
responses
.
GET
,
url
=
f
'
{
reporting_base_uri
}
/scid/current
'
,
url
=
f
'
{
reporting_base_uri
}
/scid/current
'
,
...
@@ -28,8 +28,8 @@ def test_inventory_service_download():
...
@@ -28,8 +28,8 @@ def test_inventory_service_download():
)
)
responses
.
add
(
responses
.
add
(
method
=
responses
.
GET
,
method
=
responses
.
GET
,
url
=
f
'
{
inventory_base_uri
}
/map/
services
'
,
url
=
f
'
{
inventory_base_uri
}
/map/
equipment
'
,
json
=
load_test_data
(
'
inprov-
services
.json
'
)
json
=
load_test_data
(
'
inprov-
equipment
.json
'
)
)
)
with
tempfile
.
TemporaryDirectory
()
as
tmp_dir
:
with
tempfile
.
TemporaryDirectory
()
as
tmp_dir
:
...
@@ -41,11 +41,11 @@ def test_inventory_service_download():
...
@@ -41,11 +41,11 @@ def test_inventory_service_download():
# assert os.path.exists(os.path.join(tmp_dir, services.POLLER_INTERFACES_CACHE_FILENAME))
# assert os.path.exists(os.path.join(tmp_dir, services.POLLER_INTERFACES_CACHE_FILENAME))
cached_data
=
cache
.
get
(
inventory
.
INPROV_POLLER_INTERFACES_CACHE_FILENAME
)
#
cached_data = cache.get(inventory.INPROV_POLLER_INTERFACES_CACHE_FILENAME)
assert
cached_data
==
load_test_data
(
'
poller-interfaces.json
'
)
#
assert cached_data == load_test_data('poller-interfaces.json')
cached_data
=
cache
.
get
(
inventory
.
INPROV_
MAP_SERVICES
_CACHE_FILENAME
)
cached_data
=
cache
.
get
(
inventory
.
INPROV_
EQUIPMENT
_CACHE_FILENAME
)
assert
cached_data
==
load_test_data
(
'
inprov-
services
.json
'
)
assert
cached_data
==
load_test_data
(
'
inprov-
equipment
.json
'
)
cached_data
=
cache
.
get
(
inventory
.
REPORTING_SCID_CURRENT_CACHE_FILENAME
)
cached_data
=
cache
.
get
(
inventory
.
REPORTING_SCID_CURRENT_CACHE_FILENAME
)
assert
cached_data
==
load_test_data
(
'
scid-current.json
'
)
assert
cached_data
==
load_test_data
(
'
scid-current.json
'
)
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment