diff --git a/Changelog.md b/Changelog.md
index abf10fcfcb23ac39ed7d51ac1655acb3b4b52ace..7945de885d49a323e82d380a7af0c8220a56db3a 100644
--- a/Changelog.md
+++ b/Changelog.md
@@ -1,9 +1,16 @@
# Changelog
-## [0.3] - 2025-02-13
+## [0.5] - 2025-03-14
+- Improved CSV eport functionality
+- Added Google authentication support
+- Set up PstgreSQL as the default DB
+- Imporved file export name
+- Added unittests
+
+## [0.4] - 2025-02-13
- Added CSV Export and Modify Functionality
-## [0.2] - 2025-01-27
+## [0.3] - 2025-01-27
- Added new validations
## [0.2] - 2025-01-09
diff --git a/pyproject.toml b/pyproject.toml
index 491a227d2953598e0456db844b6983c61e7fa0db..4650bd82d4c36d53aca91084f0f6a36f424b402b 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -37,4 +37,10 @@ line-length = 120
[tool.ruff.lint.flake8-tidy-imports]
ban-relative-imports = "all"
+[tool.ruff.lint.per-file-ignores]
+"test/*" = ["ARG001", "D", "S101", "PLR2004", "PLR0917", "PLR0914", "PLC0415", "PLC2701"]
+[tool.pytest.ini_options]
+DJANGO_SETTINGS_MODULE = "test.settings"
+django_find_project = false
+python_files = ["tests.py", "test_*.py", "*_tests.py"]
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
index 0fb6979ad1f0d490d1ea74e2384a40dd911b001c..a1b9380c20310dd608358bc08ff1c634d5cc92c9 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,7 +1,15 @@
-Django==5.0.11
+Django>=5.0,<5.1
+djangorestframework==3.15.2
ruff
mypy
tox
sphinx
sphinx-autodoc-typehints
mssql-django
+pytest
+pytest-django
+pytest-mock
+faker
+coverage
+social-auth-app-django==5.4.3
+psycopg[binary,pool]
\ No newline at end of file
diff --git a/sage_validation/accounts/__init__.py b/sage_validation/accounts/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/sage_validation/accounts/admin.py b/sage_validation/accounts/admin.py
new file mode 100644
index 0000000000000000000000000000000000000000..ae0e150fb8b9da27994b7720a6e93785ac716426
--- /dev/null
+++ b/sage_validation/accounts/admin.py
@@ -0,0 +1,13 @@
+"""Admin configuration for the UserActivityLog model."""
+from django.contrib import admin
+
+from sage_validation.accounts.models import UserActivityLog
+
+
+@admin.register(UserActivityLog)
+class UserActivityLogAdmin(admin.ModelAdmin):
+ """Admin configuration for the UserActivityLog model."""
+
+ list_display = ("user", "action", "name", "input_file_hash", "output_file_hash", "timestamp")
+ search_fields = ("user__username", "name", "action")
+ list_filter = ("action", "timestamp")
diff --git a/sage_validation/accounts/apps.py b/sage_validation/accounts/apps.py
new file mode 100644
index 0000000000000000000000000000000000000000..97f433049e1d7e1c46c6d8bb42fa7f41213cb535
--- /dev/null
+++ b/sage_validation/accounts/apps.py
@@ -0,0 +1,9 @@
+"""Django app configuration for the accounts' app."""
+from django.apps import AppConfig
+
+
+class AccountsConfig(AppConfig):
+ """App configuration for the accounts' app."""
+
+ default_auto_field = "django.db.models.BigAutoField"
+ name = "sage_validation.accounts"
diff --git a/sage_validation/accounts/migrations/0001_initial.py b/sage_validation/accounts/migrations/0001_initial.py
new file mode 100644
index 0000000000000000000000000000000000000000..163c095f3f495857b641c7615319b1e8ec05e749
--- /dev/null
+++ b/sage_validation/accounts/migrations/0001_initial.py
@@ -0,0 +1,55 @@
+# Generated by Django 5.0.11 on 2025-03-03 09:09
+
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+ initial = True
+
+ dependencies = [
+ migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name="UserActivityLog",
+ fields=[
+ (
+ "id",
+ models.BigAutoField(
+ auto_created=True,
+ primary_key=True,
+ serialize=False,
+ verbose_name="ID",
+ ),
+ ),
+ (
+ "action",
+ models.CharField(
+ choices=[("upload", "Upload"), ("download", "Download")],
+ max_length=10,
+ ),
+ ),
+ ("name", models.CharField(max_length=255)),
+ (
+ "input_file_hash",
+ models.CharField(blank=True, max_length=64, null=True),
+ ),
+ (
+ "output_file_hash",
+ models.CharField(blank=True, max_length=64, null=True),
+ ),
+ ("timestamp", models.DateTimeField(auto_now_add=True)),
+ (
+ "user",
+ models.ForeignKey(
+ on_delete=django.db.models.deletion.CASCADE,
+ related_name="activity_logs",
+ to=settings.AUTH_USER_MODEL,
+ ),
+ ),
+ ],
+ ),
+ ]
diff --git a/sage_validation/accounts/migrations/__init__.py b/sage_validation/accounts/migrations/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/sage_validation/accounts/models.py b/sage_validation/accounts/models.py
new file mode 100644
index 0000000000000000000000000000000000000000..40c3631433f95a58212f49014681ff6c53a75753
--- /dev/null
+++ b/sage_validation/accounts/models.py
@@ -0,0 +1,46 @@
+"""Models for the accounts app."""
+import hashlib
+from typing import ClassVar
+
+from django.contrib.auth.models import User
+from django.db import models
+
+
+class UserActivityLog(models.Model):
+ """Model to log user activities."""
+
+ ACTION_CHOICES: ClassVar[list[tuple[str, str]]] = [
+ ("upload", "Upload"),
+ ("download", "Download"),
+ ]
+
+ user = models.ForeignKey(User, on_delete=models.CASCADE, related_name="activity_logs")
+ action = models.CharField(max_length=10, choices=ACTION_CHOICES)
+ name = models.CharField(max_length=255)
+ input_file_hash = models.CharField(max_length=64, blank=True, null=True)
+ output_file_hash = models.CharField(max_length=64, blank=True, null=True)
+ timestamp = models.DateTimeField(auto_now_add=True)
+
+ def __str__(self) -> str:
+ """Return a string representation of the UserActivityLog object."""
+ return f"{self.user.username} - {self.action} - {self.name} ({self.timestamp})"
+
+ @staticmethod
+ def generate_file_hash(file_obj: object) -> str:
+ """Generate SHA-256 hash for a file-like object."""
+ hasher = hashlib.sha256()
+
+ if hasattr(file_obj, "chunks"):
+ for chunk in file_obj.chunks():
+ hasher.update(chunk)
+ elif isinstance(file_obj, str):
+ hasher.update(file_obj.encode("utf-8"))
+
+ elif isinstance(file_obj, list):
+ hasher.update(str(file_obj).encode("utf-8"))
+
+ else:
+ err = "generate_file_hash() expected a file, string, or list."
+ raise TypeError(err)
+
+ return hasher.hexdigest()
diff --git a/sage_validation/accounts/urls.py b/sage_validation/accounts/urls.py
new file mode 100644
index 0000000000000000000000000000000000000000..d2b6eada72dd075d384255e9b6bc90975172bc96
--- /dev/null
+++ b/sage_validation/accounts/urls.py
@@ -0,0 +1,7 @@
+"""URL configuration for the accounts' app."""
+from django.contrib.auth.views import LogoutView
+from django.urls import path
+
+urlpatterns = [
+ path("logout/", LogoutView.as_view(), name="logout"),
+]
diff --git a/sage_validation/accounts/views.py b/sage_validation/accounts/views.py
new file mode 100644
index 0000000000000000000000000000000000000000..73cfcebfbe2f5309c824cbc422921dd3d897e64d
--- /dev/null
+++ b/sage_validation/accounts/views.py
@@ -0,0 +1 @@
+"""Views for accounts app."""
diff --git a/sage_validation/file_validator/templates/upload.html b/sage_validation/file_validator/templates/upload.html
index 4a595da4794ec9281508073a490bc54daa4549f5..a9428905a06c54a7c936c2d18c6bcddee3c6d32a 100644
--- a/sage_validation/file_validator/templates/upload.html
+++ b/sage_validation/file_validator/templates/upload.html
@@ -40,21 +40,18 @@
</div>
</div>
-
<script>
- const form = document.getElementById('uploadForm');
- const fileInput = document.getElementById('fileInput');
- const errorSection = document.getElementById('errorSection');
- const errorList = document.getElementById('errorList');
- const successSection = document.getElementById('successSection');
- const successMessage = document.getElementById('successMessage');
- const downloadSection = document.getElementById('downloadSection');
- const downloadLink = document.getElementById('downloadLink');
-
- form.addEventListener('submit', async function (e) {
+ document.getElementById('uploadForm').addEventListener('submit', async function (e) {
e.preventDefault();
- // Clear previous messages
+ const fileInput = document.getElementById('fileInput');
+ const errorSection = document.getElementById('errorSection');
+ const errorList = document.getElementById('errorList');
+ const successSection = document.getElementById('successSection');
+ const successMessage = document.getElementById('successMessage');
+ const downloadSection = document.getElementById('downloadSection');
+ const downloadLink = document.getElementById('downloadLink');
+
errorList.innerHTML = '';
successMessage.innerHTML = '';
errorSection.classList.add('hidden');
@@ -65,7 +62,7 @@
formData.append('file', fileInput.files[0]);
try {
- const response = await fetch('', {
+ const response = await fetch("{% url 'upload-file' %}", {
method: 'POST',
body: formData,
headers: {
@@ -80,26 +77,14 @@
successSection.classList.remove('hidden');
downloadLink.href = result.download_url;
downloadSection.classList.remove('hidden');
- } else if (response.status === 400 && result.status === 'error') {
+ } else {
errorList.innerHTML = '';
- if (Array.isArray(result.errors)) {
- result.errors.forEach(errorObj => {
- if (typeof errorObj === 'string') {
- const li = document.createElement('li');
- li.textContent = errorObj;
- errorList.appendChild(li);
- } else {
- for (const [field, messages] of Object.entries(errorObj)) {
- messages.forEach(message => {
- const li = document.createElement('li');
- li.textContent = `${field}: ${message}`;
- errorList.appendChild(li);
- });
- }
- }
- });
- } else if (typeof result.errors === 'object') {
+ if (response.status === 403) {
+ const li = document.createElement('li');
+ li.textContent = 'You are not authorized to perform this action.';
+ errorList.appendChild(li);
+ } else if (response.status === 400 && result.status === 'error') {
for (const [field, messages] of Object.entries(result.errors)) {
messages.forEach(message => {
const li = document.createElement('li');
@@ -109,7 +94,7 @@
}
} else {
const li = document.createElement('li');
- li.textContent = result.errors;
+ li.textContent = 'An unexpected error occurred. Please try again.';
errorList.appendChild(li);
}
@@ -117,7 +102,7 @@
}
} catch (error) {
const li = document.createElement('li');
- li.textContent = 'An unexpected error occurred. Please try again.';
+ li.textContent = 'Failed to connect to the server. Please check your internet connection.';
errorList.appendChild(li);
errorSection.classList.remove('hidden');
}
diff --git a/sage_validation/file_validator/tests.py b/sage_validation/file_validator/tests.py
deleted file mode 100644
index 676ebd605102ad796947e51e5e7b946f30420eb3..0000000000000000000000000000000000000000
--- a/sage_validation/file_validator/tests.py
+++ /dev/null
@@ -1 +0,0 @@
-"""All the tests for the file_validator app."""
diff --git a/sage_validation/file_validator/urls.py b/sage_validation/file_validator/urls.py
index a5c0d9fd7840097233aabed1184a392037d7bcc1..ee5ae78157bbb92dcb77a93df310e41755f46739 100644
--- a/sage_validation/file_validator/urls.py
+++ b/sage_validation/file_validator/urls.py
@@ -2,9 +2,10 @@
from django.urls import path
-from sage_validation.file_validator.views import CSVExportView, CSVUploadView
+from sage_validation.file_validator.views import CSVExportAPIView, CSVUploadAPIView, upload_page_view
urlpatterns = [
- path("upload/", CSVUploadView.as_view(), name="upload-file"),
- path("export/", CSVExportView.as_view(), name="export-file"),
+ path("upload-page/", upload_page_view, name="upload-page"),
+ path("api/upload/", CSVUploadAPIView.as_view(), name="upload-file"),
+ path("api/export/", CSVExportAPIView.as_view(), name="export-file"),
]
diff --git a/sage_validation/file_validator/views.py b/sage_validation/file_validator/views.py
index 99efd175b0fa9498eb15127578cd85e272e21ac1..5eb98dddf158d5597bdad04b41b256d7b491aaad 100644
--- a/sage_validation/file_validator/views.py
+++ b/sage_validation/file_validator/views.py
@@ -1,15 +1,19 @@
"""Views for the file_validator app."""
import csv
import io
-from typing import Any
+from typing import ClassVar
-from django.http import HttpRequest, HttpResponse, JsonResponse
+from django.http import HttpRequest, HttpResponse
from django.shortcuts import render
from django.urls import reverse_lazy
from django.utils import timezone
-from django.views.generic.base import View
-from django.views.generic.edit import FormView
+from rest_framework import status
+from rest_framework.permissions import IsAuthenticated
+from rest_framework.request import Request
+from rest_framework.response import Response
+from rest_framework.views import APIView
+from sage_validation.accounts.models import UserActivityLog
from sage_validation.file_validator.forms import CSVUploadForm
from sage_validation.file_validator.models import MeoCostCentres, XxData
@@ -19,46 +23,44 @@ def index_view(request: HttpRequest) -> HttpResponse:
return render(request, "index.html")
-class CSVUploadView(FormView):
- """View for uploading a CSV file."""
+def upload_page_view(request: HttpRequest) -> HttpResponse:
+ """Render the file upload page."""
+ return render(request, "upload.html")
- template_name = "upload.html"
- form_class = CSVUploadForm
- success_url = reverse_lazy("upload-file")
- def get_context_data(self, **kwargs: dict[str, Any]) -> dict[str, Any]:
- """Render the form with no error message on GET request."""
- context = super().get_context_data(**kwargs)
- context["error"] = None
- context["message"] = None
- return context
+class CSVUploadAPIView(APIView):
+ """API view for uploading a CSV file."""
+
+ permission_classes: ClassVar[list] = [IsAuthenticated]
+
+ def post(self, request: Request) -> Response:
+ """Handle CSV upload and validation."""
+ form = CSVUploadForm(data=request.data, files=request.FILES)
+
+ if not form.is_valid():
+ return Response({"status": "error", "errors": form.errors}, status=status.HTTP_400_BAD_REQUEST)
- def form_valid(self, form: CSVUploadForm) -> JsonResponse:
- """Handle the CSV validation, store valid data, and prepare for export."""
csv_file = form.cleaned_data["file"]
csv_file.seek(0)
decoded_file = csv_file.read().decode("utf-8").strip()
if not decoded_file:
- return JsonResponse({"status": "error", "message": "Uploaded file is empty."}, status=400)
+ return Response({"status": "error", "message": "Uploaded file is empty."},
+ status=status.HTTP_400_BAD_REQUEST)
reader = csv.DictReader(io.StringIO(decoded_file))
csv_data: list[dict[str, str]] = list(reader)
updated_data = self.update_fields(csv_data)
+ request.session["validated_csv"] = updated_data
+ request.session["input_file_hash"] = UserActivityLog.generate_file_hash(csv_file)
+ request.session.modified = True
- self.request.session["validated_csv"] = updated_data
- self.request.session.modified = True
-
- return JsonResponse({
+ return Response({
"status": "success",
"message": "File successfully uploaded and processed.",
"download_url": reverse_lazy("export-file")
- })
-
- def form_invalid(self, form: CSVUploadForm) -> JsonResponse:
- """Handle the form when it is invalid."""
- return JsonResponse({"status": "error", "errors": form.errors}, status=400)
+ }, status=status.HTTP_200_OK)
@staticmethod
def update_fields(csv_data: list[dict[str, str]]) -> list[dict[str, str]]:
@@ -91,27 +93,36 @@ class CSVUploadView(FormView):
row[f"NominalAnalysisNominalAccountNumber/{repeat}"] = (
xx_data[0] if cc_type == "Project" else xx_data[1]
)
-
repeat += 1
return csv_data
-class CSVExportView(View):
- """View for exporting the updated CSV file."""
+class CSVExportAPIView(APIView):
+ """API view for exporting the updated CSV file."""
- def get(self, request: HttpRequest) -> HttpResponse:
- """Generate a downloadable CSV file with updated values."""
+ def get(self, request: Request) -> Response:
+ """Return processed CSV as a downloadable response."""
csv_data: list[dict[str, str]] = request.session.get("validated_csv", [])
+ input_file_hash: str = request.session.get("input_file_hash", "")
if not csv_data:
- return HttpResponse("No data available for export.", status=400)
-
+ return Response({"status": "error", "message": "No data available for export."},
+ status=status.HTTP_400_BAD_REQUEST)
+ file_name = f"Validated_{csv_data[0].get('TransactionReference', 'file')}.csv"
response = HttpResponse(content_type="text/csv")
- response["Content-Disposition"] = 'attachment; filename="updated_file.csv"'
+ response["Content-Disposition"] = f"attachment; filename={file_name}"
writer = csv.DictWriter(response, fieldnames=csv_data[0].keys())
writer.writeheader()
writer.writerows(csv_data)
-
+ # Log the user activity
+ UserActivityLog.objects.create(
+ user=request.user,
+ action="download",
+ name=file_name,
+ input_file_hash=input_file_hash,
+ output_file_hash=UserActivityLog.generate_file_hash(csv_data),
+ timestamp=timezone.now()
+ )
return response
diff --git a/sage_validation/settings.py b/sage_validation/settings.py
index 1a00d60d4de5129a8f1a28d43bc7160f5cea9272..904d0878f92be419b7995d2a83da0634a06a57d3 100644
--- a/sage_validation/settings.py
+++ b/sage_validation/settings.py
@@ -9,6 +9,7 @@ For the full list of settings and their values, see
https://docs.djangoproject.com/en/5.1/ref/settings/
"""
import os
+from datetime import timedelta
from pathlib import Path
# Build paths inside the project like this: BASE_DIR / 'subdir'.
@@ -18,7 +19,7 @@ BASE_DIR = Path(__file__).resolve().parent
# See https://docs.djangoproject.com/en/5.1/howto/deployment/checklist/
# SECURITY WARNING: keep the secret key used in production secret!
-SECRET_KEY = os.getenv("SECRET_KEY")
+SECRET_KEY = os.getenv("SECRET_KEY", "test-secret-key")
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True
@@ -27,6 +28,9 @@ ALLOWED_HOSTS: list[str] = os.getenv("ALLOWED_HOSTS", "").split(",")
# Application definition
INSTALLED_APPS = [
+ "rest_framework",
+ "rest_framework.authtoken",
+ "social_django",
"django.contrib.admin",
"django.contrib.auth",
"django.contrib.contenttypes",
@@ -36,6 +40,7 @@ INSTALLED_APPS = [
]
LOCAL_APPS = [
"sage_validation.file_validator",
+ "sage_validation.accounts",
]
THIRD_PARTY_APPS: list[str] = []
@@ -48,6 +53,7 @@ MIDDLEWARE = [
"django.middleware.csrf.CsrfViewMiddleware",
"django.contrib.auth.middleware.AuthenticationMiddleware",
"django.contrib.messages.middleware.MessageMiddleware",
+ "social_django.middleware.SocialAuthExceptionMiddleware",
"django.middleware.clickjacking.XFrameOptionsMiddleware",
]
@@ -64,6 +70,8 @@ TEMPLATES = [
"django.template.context_processors.request",
"django.contrib.auth.context_processors.auth",
"django.contrib.messages.context_processors.messages",
+ "social_django.context_processors.backends",
+ "social_django.context_processors.login_redirect",
],
},
},
@@ -76,20 +84,24 @@ WSGI_APPLICATION = "sage_validation.wsgi.application"
DATABASES = {
"default": {
- "ENGINE": "django.db.backends.sqlite3",
- "NAME": BASE_DIR / "db.sqlite3",
+ "ENGINE": "django.db.backends.postgresql",
+ "NAME": os.getenv("POSTGRES_DB_NAME", "sage_validation_db"),
+ "USER": os.getenv("POSTGRES_DB_USER", "sage_user"),
+ "PASSWORD": os.getenv("POSTGRES_DB_PASSWORD", "sage_password"),
+ "HOST": os.getenv("POSTGRES_DB_HOST", "localhost"),
+ "PORT": os.getenv("POSTGRES_DB_PORT", "5432"),
},
"meo": {
- 'ENGINE': 'mssql',
- 'NAME': os.getenv("MSSQL_DB_NAME", ""),
- 'USER': os.getenv("MSSQL_DB_USER", ""),
- 'PASSWORD': os.getenv("MSSQL_DB_PASSWORD", ""),
- 'HOST': os.getenv("MSSQL_DB_HOST", "localhost"),
- 'PORT': os.getenv("MSSQL_DB_PORT", ""),
+ "ENGINE": "mssql",
+ "NAME": os.getenv("MSSQL_DB_NAME", ""),
+ "USER": os.getenv("MSSQL_DB_USER", ""),
+ "PASSWORD": os.getenv("MSSQL_DB_PASSWORD", ""),
+ "HOST": os.getenv("MSSQL_DB_HOST", "localhost"),
+ "PORT": os.getenv("MSSQL_DB_PORT", ""),
- 'OPTIONS': {
- 'driver': 'ODBC Driver 18 for SQL Server',
- 'extra_params': 'TrustServerCertificate=Yes;'
+ "OPTIONS": {
+ "driver": "ODBC Driver 18 for SQL Server",
+ "extra_params": "TrustServerCertificate=Yes;"
},
},
@@ -143,6 +155,19 @@ STATICFILES_DIRS = [
]
MEDIA_ROOT = os.getenv("MEDIA_ROOT", BASE_DIR / "media")
-MEDIA_URL = '/media/'
+MEDIA_URL = "/media/"
CSRF_TRUSTED_ORIGINS = os.getenv("CSRF_TRUSTED_ORIGINS", "").split(",")
+AUTHENTICATION_BACKENDS = (
+ "social_core.backends.google.GoogleOAuth2",
+ "django.contrib.auth.backends.ModelBackend",
+)
+
+SOCIAL_AUTH_GOOGLE_OAUTH2_KEY = os.getenv("SOCIAL_AUTH_GOOGLE_OAUTH2_KEY", "")
+SOCIAL_AUTH_GOOGLE_OAUTH2_SECRET = os.getenv("SOCIAL_AUTH_GOOGLE_OAUTH2_SECRET", "")
+SOCIAL_AUTH_GOOGLE_OAUTH2_SCOPE = ["email", "profile"]
+SOCIAL_AUTH_GOOGLE_OAUTH2_REDIRECT_URI = os.getenv("SOCIAL_AUTH_GOOGLE_OAUTH2_REDIRECT_URI", "")
+SOCIAL_AUTH_JSONFIELD_ENABLED = True
+SOCIAL_AUTH_URL_NAMESPACE = "social"
+LOGIN_REDIRECT_URL = "/"
+LOGOUT_REDIRECT_URL = "/"
\ No newline at end of file
diff --git a/sage_validation/templates/base.html b/sage_validation/templates/base.html
index 20f3c23d95c69128f00d91f7a9e74061ec522528..71daa8e5dee039043a047c18e787b54780444c5f 100644
--- a/sage_validation/templates/base.html
+++ b/sage_validation/templates/base.html
@@ -18,7 +18,18 @@
<a href="{% url "index" %}" class="text-white text-2xl font-bold tracking-wide">Sage Validation</a> <!-- Made title clickable -->
</div>
<div class="flex items-center"> <!-- Login link -->
- <a href="/login" class="text-white hover:text-gray-300">Login</a>
+ {% if user.is_authenticated %}
+ <p class="text-white">Welcome, {{ user.username }}!</p>
+ <form action="{% url 'logout' %}" method="post" class="inline-block ml-4">
+ {% csrf_token %}
+ <button type="submit"
+ class="bg-red-600 text-white px-4 py-2 rounded-lg hover:bg-red-700 focus:outline-none focus:ring focus:ring-red-400">
+ Logout
+ </button>
+ </form>
+ {% else %}
+ <a href="{% url "social:begin" "google-oauth2" %}" class="text-white hover:text-gray-300">Login</a>
+ {% endif %}
<button id="mobile-menu-button" class="md:hidden text-white focus:outline-none ml-4">
<svg class="h-6 w-6" fill="none" stroke="currentColor" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M4 6h16M4 12h16m-7 6h7"></path>
diff --git a/sage_validation/templates/index.html b/sage_validation/templates/index.html
index 7d7376ea866409f6da728b1c6a343fa98b8fb7a4..72db6b8b5598643485a9e64dcfd7d9b10aa1f65b 100644
--- a/sage_validation/templates/index.html
+++ b/sage_validation/templates/index.html
@@ -8,7 +8,7 @@
<h1 class="text-5xl md:text-6xl font-bold mb-12 text-blue-900">Welcome to Sage Validation</h1>
<p class="text-xl md:text-2xl mb-16 text-gray-700">Click the button below to upload your file for validation.</p>
- <a href="{% url "upload-file" %}"
+ <a href="{% url "upload-page" %}"
class="inline-flex py-4 px-16 bg-blue-600 text-white text-lg md:text-xl font-bold rounded-full shadow-lg transition-transform transform hover:scale-105 focus:outline-none focus:ring-4 focus:ring-blue-300 focus:ring-opacity-50">
Upload File
</a>
diff --git a/sage_validation/urls.py b/sage_validation/urls.py
index af6d6158c155dd64a21c7170bb90f05d1fc7020d..1146896db11939a8c6ddc3db541093f717b62166 100644
--- a/sage_validation/urls.py
+++ b/sage_validation/urls.py
@@ -9,5 +9,6 @@ urlpatterns = [
path("admin/", admin.site.urls),
path("file-validator/", include("sage_validation.file_validator.urls")),
path("", index_view, name="index"),
-
+ path("", include("social_django.urls", namespace="social")),
+ path("accounts/", include("sage_validation.accounts.urls")),
]
diff --git a/setup.py b/setup.py
index 09c3d78cb76669e38b66c2433c81b9f3ece4497f..f4ffd2f16271a63edda19ec5a7923d13cb73ac28 100644
--- a/setup.py
+++ b/setup.py
@@ -3,12 +3,15 @@ from setuptools import find_packages, setup
setup(
name="sage-validation",
- version="0.4",
+ version="0.5",
packages=find_packages(),
include_package_data=True,
install_requires=[
- "Django==5.0.11",
+ "Django>=5.0,<5.1",
"mssql-django==1.5",
+ "djangorestframework==3.15.2",
+ "social-auth-app-django==5.4.3",
+ "psycopg[binary,pool]",
],
extras_require={
"prod": [
diff --git a/test/__init__.py b/test/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/test/conftest.py b/test/conftest.py
new file mode 100644
index 0000000000000000000000000000000000000000..94a62ed9ada92e49af102a8213b29e3866eb9b90
--- /dev/null
+++ b/test/conftest.py
@@ -0,0 +1,154 @@
+"""Fixtures for the sage_validation tests."""
+from unittest.mock import MagicMock
+
+import pytest
+from django.contrib.auth.models import User
+from django.core.files.uploadedfile import SimpleUploadedFile
+from faker import Faker
+from rest_framework.test import APIClient
+
+from sage_validation.file_validator.models import MeoCostCentres, MeoValidSuppliers, XxData
+
+
+@pytest.fixture
+def sample_input_file() -> SimpleUploadedFile:
+ """Create a sample valid CSV file for testing."""
+ csv_headers_list = [
+ "AccountNumber",
+ "CBAccountNumber",
+ "DaysDiscountValid",
+ "DiscountValue",
+ "DiscountPercentage",
+ "DueDate",
+ "GoodsValueInAccountCurrency",
+ "PurControlValueInBaseCurrency",
+ "DocumentToBaseCurrencyRate",
+ "DocumentToAccountCurrencyRate",
+ "PostedDate",
+ "QueryCode",
+ "TransactionReference",
+ "SecondReference",
+ "Source",
+ "SYSTraderTranType",
+ "TransactionDate",
+ "UniqueReferenceNumber",
+ "UserNumber",
+ "TaxValue",
+ "SYSTraderGenerationReasonType",
+ "GoodsValueInBaseCurrency",
+ "TransactionReference",
+
+ # NominalAnalysis repeating columns (Example: /1 for first occurrence)
+ "NominalAnalysisTransactionValue/1",
+ "NominalAnalysisNominalAccountNumber/1",
+ "NominalAnalysisNominalCostCentre/1",
+ "NominalAnalysisNominalDepartment/1",
+ "NominalAnalysisNominalAnalysisNarrative/1",
+ "NominalAnalysisTransactionAnalysisCode/1",
+
+ # TaxAnalysis repeating columns (Example: /1 for first occurrence)
+ "TaxAnalysisTaxRate/1",
+ "TaxAnalysisGoodsValueBeforeDiscount/1",
+ "TaxAnalysisDiscountValue/1",
+ "TaxAnalysisDiscountPercentage/1",
+ "TaxAnalysisTaxOnGoodsValue/1",
+ ]
+ csv_headers = ",".join(csv_headers_list)
+
+ csv_content_list = [
+ "12345", # AccountNumber
+ "54321", # CBAccountNumber
+ "30", # DaysDiscountValid
+ "10.5", # DiscountValue
+ "5.0", # DiscountPercentage
+ "2024-02-10", # DueDate
+ "1000", # GoodsValueInAccountCurrency
+ "950", # PurControlValueInBaseCurrency
+ "1.2", # DocumentToBaseCurrencyRate
+ "1.1", # DocumentToAccountCurrencyRate
+ "2024-02-05", # PostedDate
+ "Q1", # QueryCode
+ "TRX123", # TransactionReference
+ "SREF123", # SecondReference
+ "80", # Source
+ "4", # SYSTraderTranType
+ "2024-02-01", # TransactionDate
+ "UR123", # UniqueReferenceNumber
+ "42", # UserNumber
+ "10", # TaxValue
+ "1000", # SYSTraderGenerationReasonType
+ "1200", # GoodsValueInBaseCurrency
+ "BK123", # TransactionReference(Batch Number)
+
+ # NominalAnalysis repeating values (Example: /1)
+ "500.75", # NominalAnalysisTransactionValue/1
+ "ACC100", # NominalAnalysisNominalAccountNumber/1
+ "CC100", # NominalAnalysisNominalCostCentre/1
+ "DEP200", # NominalAnalysisNominalDepartment/1
+ "Sample Narrative", # NominalAnalysisNominalAnalysisNarrative/1
+ "TAC100", # NominalAnalysisTransactionAnalysisCode/1
+
+ # TaxAnalysis repeating values (Example: /1)
+ "20.5", # TaxAnalysisTaxRate/1
+ "900", # TaxAnalysisGoodsValueBeforeDiscount/1
+ "30", # TaxAnalysisDiscountValue/1
+ "3.5", # TaxAnalysisDiscountPercentage/1
+ "180", # TaxAnalysisTaxOnGoodsValue/1
+ ]
+ csv_content = ",".join(csv_content_list)
+
+ return SimpleUploadedFile("test.csv", f"{csv_headers}\n{csv_content}".encode(), content_type="text/csv")
+
+@pytest.fixture
+def mock_meo_database(mocker: MagicMock)-> None:
+ """Mock the meo database since it's read-only."""
+ fake = Faker()
+
+ # Mock MeoValidSuppliers
+ supplier_mock = MagicMock()
+ supplier_mock.all.return_value = [
+ MeoValidSuppliers(
+ supplier_account_number=str(fake.random_int(min=10000, max=99999)), supplier_account_name=fake.company()
+ ),
+ MeoValidSuppliers(supplier_account_number="12345", supplier_account_name="Sample Narrative")
+ ]
+ mocker.patch("sage_validation.file_validator.models.MeoValidSuppliers.objects.using", return_value=supplier_mock)
+
+ # Mock MeoCostCentres
+ cost_centre_mock = MagicMock()
+ cost_centre_mock.all.return_value = [
+ MeoCostCentres(cc="CC100", cc_type="Project", cc_name="CostCentreName", id=1),
+ MeoCostCentres(cc="CC200", cc_type="Overhead", cc_name="OverheadName", id=2),
+ MeoCostCentres(cc="CC300", cc_type="Overhead", cc_name="DepartmentName", id=3)
+ ]
+ mocker.patch("sage_validation.file_validator.models.MeoCostCentres.objects.using", return_value=cost_centre_mock)
+
+ # Mock XxData
+ xx_data_mock = MagicMock()
+ xx_data_mock.all.return_value = [
+ XxData(xx_value="N100", project="ProjectCode", overhead="OverheadCode", description=fake.sentence()),
+ XxData(xx_value="N200", project="ProjectCode", overhead="OverheadCode", description=fake.sentence())
+ ]
+ mocker.patch("sage_validation.file_validator.models.XxData.objects.using", return_value=xx_data_mock)
+
+ # Mock MeoValidSageAccounts
+ sage_account_mock = MagicMock()
+ sage_account_mock.filter.return_value.exists.return_value = True
+ mocker.patch(
+ "sage_validation.file_validator.models.MeoValidSageAccounts.objects.using", return_value=sage_account_mock
+ )
+
+ # Mock MeoNominal
+ nominal_mock = MagicMock()
+ nominal_mock.filter.return_value.exists.return_value = True
+ mocker.patch("sage_validation.file_validator.models.MeoNominal.objects.using", return_value=nominal_mock)
+
+
+@pytest.fixture
+def api_client() -> APIClient:
+ """Fixture to return Django API test client."""
+ fake = Faker()
+ user = User.objects.create_user(username=fake.user_name(), password=fake.password())
+ client = APIClient()
+ client.force_authenticate(user=user)
+ return client
diff --git a/test/settings.py b/test/settings.py
new file mode 100644
index 0000000000000000000000000000000000000000..d6f0429ddc29bffad9f2a5b59aa91de9b6850b1a
--- /dev/null
+++ b/test/settings.py
@@ -0,0 +1,15 @@
+"""Settings for running tests."""
+
+from sage_validation.settings import * # noqa: F403
+
+if os.getenv("USE_SQLITE", "false").lower() == "true": # noqa: F405
+ DATABASES = {
+ "default": {
+ "ENGINE": "django.db.backends.sqlite3",
+ "NAME": "sage_validation",
+ }
+ }
+else:
+ DATABASES = {
+ "default": DATABASES["default"],
+ }
diff --git a/test/test_file_validator/__init__.py b/test/test_file_validator/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/test/test_file_validator/test_file_validator_endpoints.py b/test/test_file_validator/test_file_validator_endpoints.py
new file mode 100644
index 0000000000000000000000000000000000000000..1d3b79f811465f9ee982d0554d467270ff4941e4
--- /dev/null
+++ b/test/test_file_validator/test_file_validator_endpoints.py
@@ -0,0 +1,90 @@
+from unittest.mock import MagicMock
+
+import pytest
+from django.core.files.uploadedfile import SimpleUploadedFile
+from django.urls.base import reverse
+from rest_framework.test import APIClient
+
+from sage_validation.accounts.models import UserActivityLog
+
+UPLOAD_FILE_URL = reverse("upload-file")
+
+
+@pytest.mark.django_db
+def test_csv_upload_unauthenticated(sample_input_file: SimpleUploadedFile) -> None:
+ """Test that a valid CSV upload succeeds."""
+ api_client = APIClient()
+ response = api_client.post(UPLOAD_FILE_URL, {"file": sample_input_file}, format="multipart")
+
+ assert response.status_code == 403
+ assert response.json()["detail"] == "Authentication credentials were not provided."
+
+@pytest.mark.django_db
+def test_csv_upload_valid(
+ api_client: APIClient, sample_input_file: SimpleUploadedFile, mock_meo_database: MagicMock
+) -> None:
+ """Test that a valid CSV upload succeeds."""
+ response = api_client.post(UPLOAD_FILE_URL, {"file": sample_input_file}, format="multipart")
+
+ assert response.status_code == 200
+ assert response.json()["status"] == "success"
+ assert "download_url" in response.json()
+
+
+@pytest.mark.django_db
+def test_csv_upload_invalid_extension(api_client: APIClient) -> None:
+ """Test that uploading a non-CSV file fails."""
+ bad_file = SimpleUploadedFile("test.txt", b"Invalid content", content_type="text/plain")
+
+ response = api_client.post(UPLOAD_FILE_URL, {"file": bad_file}, format="multipart")
+
+ assert response.status_code == 400
+ assert response.json()["status"] == "error"
+ assert "errors" in response.json()
+
+
+@pytest.mark.django_db
+def test_csv_export_with_data(api_client: APIClient) -> None:
+ """Test exporting a processed CSV file."""
+ url = reverse("export-file")
+
+ # Simulate session data
+ session = api_client.session
+ session["validated_csv"] = [
+ {"AccountNumber": "12345", "TransactionDate": "01/03/2024", "TransactionReference": "BK1234"},
+ ]
+ session["input_file_hash"] = "123456"
+
+ session.save()
+
+ response = api_client.get(url)
+
+ assert UserActivityLog.objects.count() == 1
+ assert response.status_code == 200
+ assert response["Content-Disposition"] == "attachment; filename=Validated_BK1234.csv"
+ assert b"AccountNumber,TransactionDate,TransactionReference" in response.content
+ assert b"12345,01/03/2024,BK1234" in response.content
+
+
+@pytest.mark.django_db
+def test_activity_log_creation_on_csv_export(api_client: APIClient) -> None:
+ """Test that a UserActivityLog is created when exporting a CSV file."""
+ url = reverse("export-file")
+
+ # Simulate session data
+ session = api_client.session
+ session["validated_csv"] = [
+ {"AccountNumber": "12345", "TransactionDate": "01/03/2024", "TransactionReference": "BK1234"},
+ ]
+ session["input_file_hash"] = "123456"
+
+ session.save()
+
+ response = api_client.get(url)
+
+ assert response.status_code == 200
+ log = UserActivityLog.objects.first()
+ assert log.action == "download"
+ assert log.input_file_hash == "123456"
+ assert log.output_file_hash == UserActivityLog.generate_file_hash(session["validated_csv"])
+ assert log.name == "Validated_BK1234.csv"
diff --git a/test/test_file_validator/test_forms.py b/test/test_file_validator/test_forms.py
new file mode 100644
index 0000000000000000000000000000000000000000..b5df1089c1430e9a898b62cd8449ef849415f422
--- /dev/null
+++ b/test/test_file_validator/test_forms.py
@@ -0,0 +1,94 @@
+"""Tests for the file_validator forms."""
+import csv
+import io
+from unittest.mock import MagicMock
+
+from django.core.files.uploadedfile import SimpleUploadedFile
+
+from sage_validation.file_validator.forms import CSVUploadForm
+
+
+def create_modified_csv(sample_file: SimpleUploadedFile, modifications: dict[str, str]) -> SimpleUploadedFile:
+ """
+ Modify specific fields in the first row of a CSV file and return a new SimpleUploadedFile.
+
+ Args:
+ sample_file (SimpleUploadedFile): The original CSV file.
+ modifications (dict): Dictionary of column names to modified values.
+
+ Returns:
+ SimpleUploadedFile: The modified CSV file.
+ """
+ csv_content = sample_file.read().decode("utf-8").splitlines()
+ reader = csv.DictReader(csv_content)
+ rows = list(reader)
+
+ for key, value in modifications.items():
+ rows[0][key] = value
+
+ output = io.StringIO()
+ writer = csv.DictWriter(output, fieldnames=reader.fieldnames or [])
+ writer.writeheader()
+ writer.writerows(rows)
+
+ return SimpleUploadedFile("test_modified.csv", output.getvalue().encode("utf-8"), content_type="text/csv")
+
+
+def test_valid_csv_upload(sample_input_file: SimpleUploadedFile, mock_meo_database: MagicMock) -> None:
+ """Test CSV upload with valid data."""
+ form = CSVUploadForm(files={"file": sample_input_file})
+ assert form.is_valid(), f"Form errors: {form.errors}"
+
+
+def test_invalid_file_extension() -> None:
+ """Test form rejects non-CSV files."""
+ bad_file = SimpleUploadedFile("test.txt", b"Some text content", content_type="text/plain")
+ form = CSVUploadForm(files={"file": bad_file})
+ assert not form.is_valid()
+ assert "File must be in CSV format." in form.errors["file"]
+
+
+def test_missing_required_columns() -> None:
+ """Test form rejects CSV missing required headers."""
+ invalid_csv = SimpleUploadedFile("test.csv", b"AccountNumber,CBAccountNumber\n12345,54321", content_type="text/csv")
+ form = CSVUploadForm(files={"file": invalid_csv})
+ assert not form.is_valid()
+ assert "Missing required columns" in str(form.errors)
+
+
+def test_source_and_trader_type_validation(sample_input_file: SimpleUploadedFile, mock_meo_database: MagicMock) -> None:
+ """Test validation for Source and SYSTraderTranType columns."""
+ modified_file = create_modified_csv(sample_input_file, {"Source": "90", "SYSTraderTranType": "5"})
+ form = CSVUploadForm(files={"file": modified_file})
+ assert not form.is_valid()
+ assert "Row 1: 'Source' must be 80" in form.errors["file"][0]
+ assert "Row 1: 'SYSTraderTranType' must be 4" in form.errors["file"][1]
+
+
+def test_validate_nominal_analysis_account(sample_input_file: SimpleUploadedFile, mock_meo_database: MagicMock) -> None:
+ """Test validation for nominal analysis account."""
+ modified_file = create_modified_csv(sample_input_file,
+ {"NominalAnalysisNominalAnalysisNarrative/1": "Invalid Name"})
+ form = CSVUploadForm(files={"file": modified_file})
+ assert not form.is_valid()
+ assert form.errors["file"][0] == (
+ "Row 1: 'AccountNumber' must match 'Sample Narrative' in 'NominalAnalysisNominalAnalysisNarrative/1'"
+ ", but found 'Invalid Name'.")
+
+
+def test_validate_nc_cc_dep_combination_against_meo_sage_account(
+ sample_input_file: SimpleUploadedFile, mock_meo_database: MagicMock
+) -> None:
+ """Test validation for nominal analysis fields against MEO valid Sage accounts."""
+ modified_file = create_modified_csv(
+ sample_input_file,
+ {
+ "NominalAnalysisNominalCostCentre/1": "Invalid_CC",
+ "NominalAnalysisNominalAccountNumber/1": "Invalid_Account"
+ })
+
+ form = CSVUploadForm(files={"file": modified_file})
+ assert not form.is_valid()
+ assert ("Row 1: 'NominalAnalysisNominalCostCentre/1' (Invalid_CC) is not a valid cost centre."
+ in str(form.errors["file"][0]))
+
diff --git a/tox.ini b/tox.ini
index 91d0161e6590165ecf51bec47ab84ff68718aacd..1db7951f22971976c85cee2b4ecf9259839e4724 100644
--- a/tox.ini
+++ b/tox.ini
@@ -5,6 +5,17 @@ envlist = py311
deps =
mypy
ruff
+ -r requirements.txt
+
+setenv =
+ USE_SQLITE = true
+
commands =
ruff check .
mypy .
+ coverage erase
+ coverage run --source sage_validation -m pytest
+ coverage report --fail-under=90
+ coverage xml
+ coverage html
+