added MISP user managemet

config.py

@@ -9,6 +9,8 @@ VARIABLES_FILE = os.path.join(SOCTOOLS_BASE, "group_vars/all/variables.yml")
 CA_CERT_FILE = os.path.join(SOCTOOLS_BASE, "secrets/CA/ca.crt")
 KEYCLOAK_ADMIN_PASSWORD_FILE = os.path.join(SOCTOOLS_BASE, "secrets/passwords/keykloak_admin") # Note: should be keycloak, not keykloak
 
+MISP_API_KEY_FILE = os.path.join(SOCTOOLS_BASE, "secrets/tokens/misp")
+
 # Credentials of the special user for account management
 # Cert and key should be in .pem format, unencrypted
 MGMT_USER_NAME = "soctools-user-mgmt"
@@ -26,3 +28,5 @@ SOCTOOLSPROXY = None
 KEYCLOAK_BASE_URL = None
 KEYCLOAK_USERS_URL = None
 KEYCLOAK_ADMIN_PASSWORD = None
+
+MISP_API_KEY = None

main.py

@@ -3,7 +3,7 @@ import sys
 from datetime import datetime, timezone
 import os.path
 import re
-from typing import List, Dict, Optional
+from typing import List, Dict, Optional, Union
 
 from flask import Flask, render_template, request, make_response, redirect, flash
 from flask_wtf import FlaskForm
@@ -14,8 +14,9 @@ import requests
 import yaml
 from dataclasses import dataclass, field # external package - backport of Py3.7 dataclasses to Py3.6
 
-from nifi import *
 import config
+from nifi import *
+from misp import *
 
 app = Flask(__name__)
 app.secret_key = "ASDF1234 - CHANGE ME!"
@@ -33,12 +34,20 @@ def load_config():
     config.KEYCLOAK_USERS_URL = config.KEYCLOAK_BASE_URL + "/auth/admin/realms/SOCTOOLS1/users"
     # Load API key for Keycloak
     config.KEYCLOAK_ADMIN_PASSWORD = open(config.KEYCLOAK_ADMIN_PASSWORD_FILE, "r").read(100).strip() # read max 100 B, the key should never be so long
+    # Load API key for MISP
+    config.MISP_API_KEY = open(config.MISP_API_KEY_FILE, "r").read(100).strip() # read max 100 B, the key should never be so long
+
     print(f"Config loaded:\nSOCTOOLSPROXY={config.SOCTOOLSPROXY}\nKEYCLOAK_BASE_URL={config.KEYCLOAK_BASE_URL}\n"
-          f"KEYCLOAK_ADMIN_PASSWORD={config.KEYCLOAK_ADMIN_PASSWORD[:3]}...{config.KEYCLOAK_ADMIN_PASSWORD[-4:]}")
+          f"KEYCLOAK_ADMIN_PASSWORD={config.KEYCLOAK_ADMIN_PASSWORD[:3]}...{config.KEYCLOAK_ADMIN_PASSWORD[-4:]}\n"
+          f"MISP_API_KEY={config.MISP_API_KEY[:3]}...{config.MISP_API_KEY[-4:]}\n")
 
 
 # *** Custom Jinja filters ***
-def ts_to_str(ts: float) -> str:
+def ts_to_str(ts: Union[float,datetime,None]) -> str:
+    if ts is None:
+        return ""
+    if isinstance(ts, datetime):
+        return ts.replace(tzinfo=None).isoformat(sep=" ")
     return datetime.utcfromtimestamp(int(ts)).isoformat(sep=" ") # TODO Do Keycloak really use UTC timestamps?
 
 app.jinja_env.filters["ts_to_str"] = ts_to_str
@@ -252,6 +261,7 @@ def main():
             u.internal = True
     #print(users)
 
+    # ===============
     # Load NiFi users
     try:
         nifi_users = nifi_get_users()
@@ -265,6 +275,20 @@ def main():
     # List of usernames only (for easier cross-check with Keycloak users)
     nifi_usernames = set(nu["name"] for nu in nifi_users)
 
+    # ===============
+    # Load MISP users
+    try:
+        misp_users = misp_get_users()
+    except MISPError as e:
+        flash(f"ERROR: {e}", "error")
+        misp_users = []
+    # Mark "internal" users
+    for u in misp_users:
+        if u["email"] == "admin@admin.test":
+            u["internal"] = True
+    # List of usernames only (for easier cross-check with Keycloak users)
+    misp_emails = set(mu["email"] for mu in misp_users)
+
     return render_template("main.html", **locals())
 
 
@@ -283,17 +307,28 @@ def add_user():
         # Keycloak
         try:
             kc_add_user(user)
-            flash(f'User "{form_user.username.data}" successfully created in Keycloak.', "success")
+            flash(f'User "{user.username}" successfully created in Keycloak.', "success")
         except Exception as e:
             flash(f'Error when creating user in Keycloak: {e}', "error")
+
         # NiFi
         try:
             nifi_add_user(user)
-            flash(f'User "{form_user.username.data}" successfully created in NiFi.', "success")
+            flash(f'User "{user.username}" successfully created in NiFi.', "success")
         except NifiUserExistsError:
             flash(f'User "{user.username}" already exists in NiFi, nothing has changed.', "warning")
         except Exception as e:
             flash(f'Error when creating user in NiFi: {e}', "error")
+
+        # MISP
+        try:
+            misp_add_user(user)
+            flash(f'User "{user.email}" successfully created in MISP.', "success")
+        except MISPUserExistsError: # TODO
+            flash(f'User with email "{user.email}" already exists in MISP, nothing has changed.', "warning")
+        except Exception as e:
+            flash(f'Error when creating user in MISP: {e}', "error")
+
         return redirect("/") # Success - go back to main page
 
     return render_template("add_edit_user.html", form_user=form_user, user=None)
@@ -302,31 +337,54 @@ def add_user():
 @app.route("/edit_user/<username>", methods=["GET", "POST"])
 def edit_user(username: str):
     """Edit existing user. On GET show user details, on POST update user params with new values."""
-    keycloak_id = kc_get_user_by_name(username).kcid # TODO catch exception
+    try:
+        user = kc_get_user_by_name(username) # TODO catch exception
+    except KeycloakError as e:
+        flash(f'ERROR: {e}', "error")
+        return redirect('/')
+    keycloak_id = user.kcid
+    # POST = perform the update
     if request.method == "POST":
         form_user = AddUserForm() # use data from POST request
         if form_user.validate_on_submit():
             # Form submitted and valid - perform account update
-            user = UserAccount(username=form_user.username.data,
+            new_user = UserAccount(username=form_user.username.data,
                                email=form_user.email.data,
                                firstname=form_user.firstname.data,
                                lastname=form_user.lastname.data,
                                cn=form_user.cn.data,
                                dn=f"CN={form_user.cn.data}",
                                kcid=keycloak_id)
+            # Keycloak
             try:
-                kc_update_user(user)
-                flash(f'User "{form_user.username.data}" successfully updated.', "success")
-                return redirect("/") # Success - go back to main page
+                kc_update_user(new_user)
+                flash(f'User "{new_user.username}" successfully updated in Keycloak.', "success")
             except KeycloakError as e:
-                flash(f'Error when updating user: {e}', "error")
+                flash(f'Error when updating user in Keycloak: {e}', "error")
+
+            # NiFi
+            # There's just username in NiFi, no other parameters, so there's nothing to edit
+
+            # MISP
+            # Only email can be changed, other user params are not stored in MISP
+            if user.email != new_user.email:
+                try:
+                    misp_edit_user(user.email, new_user.email)
+                    flash(f"User's email successfully updated in MISP.", "success")
+                except MISPUserNotFoundError:
+                    flash(f'Error when updating user in MISP: User with email "{user.email}" not found', "error")
+                except MISPUserExistsError:
+                    flash(f'Error when updating user in MISP: User with email "{new_user.email}" already exists. '
+                          'BEWARE: An inconsistency in user accounts in Keycloak and MISP was probably just introduced '
+                          'which needs to be fixed manually in the administration of the individual services!', "error")
+                except Exception as e:
+                    flash(f'Error when updating user in MISP: {e}', "error")
+
+            return redirect("/") # Success - go back to main page
+
+        # data not valid - show form again
         return render_template("add_edit_user.html", form_user=form_user, user={"kcid": keycloak_id})
-    # else - method="GET"
-    try:
-        user = kc_get_user_by_id(keycloak_id)
-    except KeycloakError as e:
-        flash(f'ERROR: {e}', "error")
-        return redirect('/')
+    # GET = show the edit form
     form_user = AddUserForm(obj=user)
     return render_template("add_edit_user.html", form_user=form_user, user=user)
 
@@ -335,12 +393,19 @@ def edit_user(username: str):
 def delete_user(username: str):
     """Delete user given by username and redirect back to main page"""
     try:
-        keycloak_id = kc_get_user_by_name(username).kcid
-        kc_delete_user(keycloak_id)
+        user_spec = kc_get_user_by_name(username)
+    except KeycloakError as e:
+        flash(f"Error: Can't get user info from KeyCloak: {e}", "error")
+        return redirect("/")
+
+    # Keycloak
+    try:
+        kc_delete_user(user_spec.kcid)
         flash('User successfully deleted from KeyCloak.', "success")
     except KeycloakError as e:
         flash(f'Error when deleting user from KeyCloak: {e}', "error")
 
+    # NiFi
     try:
         nifi_delete_user(username)
         flash(f'User "{username}" successfully deleted from NiFi.', "success")
@@ -349,12 +414,25 @@ def delete_user(username: str):
     except NifiError as e:
         flash(f'Error when deleting user from NiFi: {e}', "error")
 
+    # MISP
+    try:
+        misp_delete_user(user_spec.email)
+        flash(f'User "{user_spec.email}" successfully deleted from MISP.', "success")
+    except MISPUserNotFoundError:
+        flash(f'User "{user_spec.email}" was not found in MISP, nothing has changed.', "warning")
+    except MISPError as e:
+        flash(f'Error when deleting user from MISP: {e}', "error")
+
     return redirect("/")
 
 
 # TODO certificates??
 
 # TODO other services (besides Keycloak)
+#  - NiFi - DONE
+#  - MISP - DONE
+#  - Kibana?
+#  - TheHive + Cortex
 
 # TODO authentication/authorization to this GUI
 

nifi.py

@@ -7,7 +7,6 @@ from operator import itemgetter
 import urllib.parse
 
 import config
-config.SOCTOOLSPROXY = "gn4soctools3.liberouter.org"
 
 # URL to initial login process
 NIFI_LOGIN_URL = "https://{soctools_proxy}:9443/nifi/login"

templates/main.html

@@ -4,8 +4,7 @@
 <p><a href="{{ url_for("add_user") }}">{{ icon('plus-circle', size="1em") }} Add new user ...</a></p>
 
 <table>
-<tr><th>Username</th><th>First name</th><th>Last name</th><th>email</th><th>CN</th><th>DN</th><th>Time created (UTC)</th><th>NiFi</th><th></th>
-
+<tr><th>Username</th><th>First name</th><th>Last name</th><th>email</th><th>CN</th><th>DN</th><th>Time created (UTC)</th><th>NiFi</th><th>MISP</th><th></th>
 {% for user in users %}
 <tr{% if user.internal %} class="internal-user"{% endif %}>
 <td>{{ user.username }}</td>
@@ -14,8 +13,9 @@
 <td>{{ user.email }}</td>
 <td>{{ user.cn }}</td>
 <td>{{ user.dn }}</td>
-<td>{{ user.ts_created.isoformat() }}</td>
+<td>{{ user.ts_created|ts_to_str }}</td>
 <td>{{ icon('check' if user.username in nifi_usernames else 'close') }}</td>
+<td>{{ icon('check' if user.email in misp_emails else 'close') }}</td>
 <td>
 {% if not user.internal -%}
 <a href="{{ url_for('edit_user', username=user.username) }}" title="Edit user">{{ icon('pencil') }}</a>
@@ -43,4 +43,21 @@
 {% endfor %}
 </table>
 
+
+<h3>MISP</h3>
+<table>
+<tr><th>Email (username)</th><th>ID</th><th>Organization</th><th>Role</th><th>Created</th><th>Last login</th>
+
+{% for user in misp_users %}
+<tr{% if user.internal %} class="internal-user"{% endif %}>
+<td>{{ user.email }}</td>
+<td>{{ user.id }}</td>
+<td>{{ user.org }}</td>
+<td>{{ user.role }}</td>
+<td>{{ user.created|ts_to_str }}</td>
+<td>{{ user.last_login|ts_to_str }}</td>
+</tr>
+{% endfor %}
+</table>
+
 {% endblock %}
\ No newline at end of file