diff --git a/mdserver.py b/mdserver.py index fbbb34753cd960d8ffda91acb10e43bfa4c55308..1ed376d672a00fdaa8ce4f41870f8176fc7091c0 100755 --- a/mdserver.py +++ b/mdserver.py @@ -1,10 +1,10 @@ #!/usr/bin/env python -from utils import read_config, Resource, Server +from utils import read_config, Resource from flask import Flask, Response config = read_config('mdserver.yaml') app = Flask(__name__) -server = Server() +server = {} @app.route('/<domain>/entities/<path:entity_id>', @@ -29,8 +29,6 @@ for domain, values in config.items(): location = values['metadir'] signer = values['signer'] server[domain] = Resource(location, signer) - server.add_watch(domain, location) - if __name__ == "__main__": app.run(host='127.0.0.1', port=5001, debug=False) diff --git a/utils.py b/utils.py index d3eb6f6a8aca373ccdc8505b78c15793c2feaa16..1e56299a58438526ee903ba6dac5f14321019a48 100755 --- a/utils.py +++ b/utils.py @@ -9,9 +9,6 @@ import yaml import pyinotify from signers import Signers -# watch_list = {} -# watch_manager = pyinotify.WatchManager() - def read_config(config): with open(config) as f: @@ -33,28 +30,69 @@ class Entity: self.cache_duration = 0 +class EventProcessor(pyinotify.ProcessEvent): + def __init__(self, resource): + self.resource = resource + + def process_default(self, event): + if event.maskname in ["IN_DELETE", + "IN_MOVED_FROM", + "IN_MOVED_TO"]: + self.resource.walk_location(event.path) + else: + self.resource.read_metadata(event.pathname) + + class Resource: + watch_list = {} + def __init__(self, location, signer): self.idps = {} self.mdfiles = {} self.signer = Signers(signer) + + self.md_watcher = pyinotify.WatchManager() + self.md_events = pyinotify.ThreadedNotifier(self.md_watcher, + EventProcessor(self)) + self.md_events.start() + self.walk_location(location) def walk_location(self, location): + old_mdfiles = self.mdfiles.copy() + location = os.path.realpath(location) + self.md_watcher.add_watch(location, pyinotify.IN_CLOSE_WRITE + | pyinotify.IN_DELETE + | pyinotify.IN_MOVED_FROM + | pyinotify.IN_MOVED_TO) + files = os.listdir(location) for file in files: mdfile = os.path.realpath(os.path.join(location, file)) if os.path.isfile(mdfile): - self.mdfiles[mdfile] = [] - self._read_metadata(mdfile) - - def _read_metadata(self, mdfile): + old_mdfiles.pop(mdfile, None) + self.read_metadata(mdfile) + + for mdf, idps in old_mdfiles.items(): + print("\n--- REMOVE METADATA --") + print(mdf) + for idp in idps: + print(f" {{sha1}}{idp}") + self.idps.pop(idp, None) + self.__dict__.pop(idp, None) + + def read_metadata(self, mdfile): print("\n--- READ METADATA --") print(mdfile) found = 0 removed = 0 - old_idps = self.mdfiles[mdfile].copy() - root = ET.ElementTree(file=mdfile).getroot() + old_idps = self.mdfiles.get(mdfile, []).copy() + mdfiles = [] + try: + root = ET.ElementTree(file=mdfile).getroot() + except Exception: + print("Invalid metadata") + return ns = root.nsmap.copy() ns['xml'] = 'http://www.w3.org/XML/1998/namespace' validUntil = root.get('validUntil') @@ -76,15 +114,16 @@ class Resource: valid_until) self.idps[sha1] = entity self.__dict__.pop(sha1, None) - if sha1 in self.mdfiles[mdfile]: - self.mdfiles[mdfile].remove(sha1) + if sha1 in old_idps: + old_idps.remove(sha1) + mdfiles.append(sha1) found += 1 for idp in old_idps: self.idps.pop(idp, None) self.__dict__.pop(idp, None) removed += 1 - self.mdfiles[mdfile] = self.idps.keys() + self.mdfiles[mdfile] = list(mdfiles) print(f"Found: {found} entities") print(f"Removed: {removed} entities") @@ -125,38 +164,3 @@ class Resource: print(f"serve {sha1}") return data - - -class EventProcessor(pyinotify.ProcessEvent): - def __init__(self, server): - self.server = server - - def process_IN_CLOSE_WRITE(self, event): - self.server.process(event.path) - - -class Server: - watch_list = {} - - def __init__(self): - self.watch_manager = pyinotify.WatchManager() - self.event_notifier = pyinotify.ThreadedNotifier(self.watch_manager, - EventProcessor(self)) - self.event_notifier.start() - - def add_watch(self, domain, location): - self.watch_list[location] = domain - self.watch_manager.add_watch(location, pyinotify.IN_CLOSE_WRITE) - - def process(self, location): - domain = self.watch_list[location] - print(f"Notify {domain} {location}") - self.__dict__[domain].walk_location(location) - - def __setitem__(self, domain, resource): - self.__dict__[domain] = resource - # watch_list[location] = domain - # watch_manager.add_watch(location, pyinotify.IN_CLOSE_WRITE) - - def __getitem__(self, domain): - return self.__dict__[domain]