diff --git a/pyproject.toml b/pyproject.toml index 491a227d2953598e0456db844b6983c61e7fa0db..4650bd82d4c36d53aca91084f0f6a36f424b402b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,4 +37,10 @@ line-length = 120 [tool.ruff.lint.flake8-tidy-imports] ban-relative-imports = "all" +[tool.ruff.lint.per-file-ignores] +"test/*" = ["ARG001", "D", "S101", "PLR2004", "PLR0917", "PLR0914", "PLC0415", "PLC2701"] +[tool.pytest.ini_options] +DJANGO_SETTINGS_MODULE = "test.settings" +django_find_project = false +python_files = ["tests.py", "test_*.py", "*_tests.py"] \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 0fb6979ad1f0d490d1ea74e2384a40dd911b001c..4edebd6b0646bd3afbc9d5f613964dea13a26e24 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,13 @@ Django==5.0.11 +django-rest-framework ruff mypy tox sphinx sphinx-autodoc-typehints mssql-django +pytest +pytest-django +pytest-mock +faker +coverage diff --git a/sage_validation/file_validator/templates/upload.html b/sage_validation/file_validator/templates/upload.html index 4a595da4794ec9281508073a490bc54daa4549f5..ec8619dd40130d1555c59dc7a549c6c359c57edc 100644 --- a/sage_validation/file_validator/templates/upload.html +++ b/sage_validation/file_validator/templates/upload.html @@ -40,21 +40,18 @@ </div> </div> - <script> - const form = document.getElementById('uploadForm'); - const fileInput = document.getElementById('fileInput'); - const errorSection = document.getElementById('errorSection'); - const errorList = document.getElementById('errorList'); - const successSection = document.getElementById('successSection'); - const successMessage = document.getElementById('successMessage'); - const downloadSection = document.getElementById('downloadSection'); - const downloadLink = document.getElementById('downloadLink'); - - form.addEventListener('submit', async function (e) { + document.getElementById('uploadForm').addEventListener('submit', async function (e) { e.preventDefault(); - // Clear previous messages + const fileInput = document.getElementById('fileInput'); + const errorSection = document.getElementById('errorSection'); + const errorList = document.getElementById('errorList'); + const successSection = document.getElementById('successSection'); + const successMessage = document.getElementById('successMessage'); + const downloadSection = document.getElementById('downloadSection'); + const downloadLink = document.getElementById('downloadLink'); + errorList.innerHTML = ''; successMessage.innerHTML = ''; errorSection.classList.add('hidden'); @@ -65,7 +62,7 @@ formData.append('file', fileInput.files[0]); try { - const response = await fetch('', { + const response = await fetch("{% url 'upload-file' %}", { method: 'POST', body: formData, headers: { @@ -81,38 +78,13 @@ downloadLink.href = result.download_url; downloadSection.classList.remove('hidden'); } else if (response.status === 400 && result.status === 'error') { - errorList.innerHTML = ''; - - if (Array.isArray(result.errors)) { - result.errors.forEach(errorObj => { - if (typeof errorObj === 'string') { - const li = document.createElement('li'); - li.textContent = errorObj; - errorList.appendChild(li); - } else { - for (const [field, messages] of Object.entries(errorObj)) { - messages.forEach(message => { - const li = document.createElement('li'); - li.textContent = `${field}: ${message}`; - errorList.appendChild(li); - }); - } - } + for (const [field, messages] of Object.entries(result.errors)) { + messages.forEach(message => { + const li = document.createElement('li'); + li.textContent = `${field}: ${message}`; + errorList.appendChild(li); }); - } else if (typeof result.errors === 'object') { - for (const [field, messages] of Object.entries(result.errors)) { - messages.forEach(message => { - const li = document.createElement('li'); - li.textContent = `${field}: ${message}`; - errorList.appendChild(li); - }); - } - } else { - const li = document.createElement('li'); - li.textContent = result.errors; - errorList.appendChild(li); } - errorSection.classList.remove('hidden'); } } catch (error) { diff --git a/sage_validation/file_validator/urls.py b/sage_validation/file_validator/urls.py index a5c0d9fd7840097233aabed1184a392037d7bcc1..ee5ae78157bbb92dcb77a93df310e41755f46739 100644 --- a/sage_validation/file_validator/urls.py +++ b/sage_validation/file_validator/urls.py @@ -2,9 +2,10 @@ from django.urls import path -from sage_validation.file_validator.views import CSVExportView, CSVUploadView +from sage_validation.file_validator.views import CSVExportAPIView, CSVUploadAPIView, upload_page_view urlpatterns = [ - path("upload/", CSVUploadView.as_view(), name="upload-file"), - path("export/", CSVExportView.as_view(), name="export-file"), + path("upload-page/", upload_page_view, name="upload-page"), + path("api/upload/", CSVUploadAPIView.as_view(), name="upload-file"), + path("api/export/", CSVExportAPIView.as_view(), name="export-file"), ] diff --git a/sage_validation/file_validator/views.py b/sage_validation/file_validator/views.py index 99efd175b0fa9498eb15127578cd85e272e21ac1..bfa80c2cde6e71b61c4849f7183a669d74b739c2 100644 --- a/sage_validation/file_validator/views.py +++ b/sage_validation/file_validator/views.py @@ -1,14 +1,15 @@ """Views for the file_validator app.""" import csv import io -from typing import Any -from django.http import HttpRequest, HttpResponse, JsonResponse +from django.http import HttpRequest, HttpResponse from django.shortcuts import render from django.urls import reverse_lazy from django.utils import timezone -from django.views.generic.base import View -from django.views.generic.edit import FormView +from rest_framework import status +from rest_framework.request import Request +from rest_framework.response import Response +from rest_framework.views import APIView from sage_validation.file_validator.forms import CSVUploadForm from sage_validation.file_validator.models import MeoCostCentres, XxData @@ -19,46 +20,41 @@ def index_view(request: HttpRequest) -> HttpResponse: return render(request, "index.html") -class CSVUploadView(FormView): - """View for uploading a CSV file.""" +def upload_page_view(request: HttpRequest) -> HttpResponse: + """Render the file upload page.""" + return render(request, "upload.html") - template_name = "upload.html" - form_class = CSVUploadForm - success_url = reverse_lazy("upload-file") - def get_context_data(self, **kwargs: dict[str, Any]) -> dict[str, Any]: - """Render the form with no error message on GET request.""" - context = super().get_context_data(**kwargs) - context["error"] = None - context["message"] = None - return context +class CSVUploadAPIView(APIView): + """API view for uploading a CSV file.""" + + def post(self, request: Request) -> Response: + """Handle CSV upload and validation.""" + form = CSVUploadForm(data=request.data, files=request.FILES) + + if not form.is_valid(): + return Response({"status": "error", "errors": form.errors}, status=status.HTTP_400_BAD_REQUEST) - def form_valid(self, form: CSVUploadForm) -> JsonResponse: - """Handle the CSV validation, store valid data, and prepare for export.""" csv_file = form.cleaned_data["file"] csv_file.seek(0) decoded_file = csv_file.read().decode("utf-8").strip() if not decoded_file: - return JsonResponse({"status": "error", "message": "Uploaded file is empty."}, status=400) + return Response({"status": "error", "message": "Uploaded file is empty."}, + status=status.HTTP_400_BAD_REQUEST) reader = csv.DictReader(io.StringIO(decoded_file)) csv_data: list[dict[str, str]] = list(reader) updated_data = self.update_fields(csv_data) + request.session["validated_csv"] = updated_data + request.session.modified = True - self.request.session["validated_csv"] = updated_data - self.request.session.modified = True - - return JsonResponse({ + return Response({ "status": "success", "message": "File successfully uploaded and processed.", "download_url": reverse_lazy("export-file") - }) - - def form_invalid(self, form: CSVUploadForm) -> JsonResponse: - """Handle the form when it is invalid.""" - return JsonResponse({"status": "error", "errors": form.errors}, status=400) + }, status=status.HTTP_200_OK) @staticmethod def update_fields(csv_data: list[dict[str, str]]) -> list[dict[str, str]]: @@ -91,21 +87,21 @@ class CSVUploadView(FormView): row[f"NominalAnalysisNominalAccountNumber/{repeat}"] = ( xx_data[0] if cc_type == "Project" else xx_data[1] ) - repeat += 1 return csv_data -class CSVExportView(View): - """View for exporting the updated CSV file.""" +class CSVExportAPIView(APIView): + """API view for exporting the updated CSV file.""" - def get(self, request: HttpRequest) -> HttpResponse: - """Generate a downloadable CSV file with updated values.""" + def get(self, request: Request) -> Response: + """Return processed CSV as a downloadable response.""" csv_data: list[dict[str, str]] = request.session.get("validated_csv", []) if not csv_data: - return HttpResponse("No data available for export.", status=400) + return Response({"status": "error", "message": "No data available for export."}, + status=status.HTTP_400_BAD_REQUEST) response = HttpResponse(content_type="text/csv") response["Content-Disposition"] = 'attachment; filename="updated_file.csv"' diff --git a/sage_validation/settings.py b/sage_validation/settings.py index 1a00d60d4de5129a8f1a28d43bc7160f5cea9272..353594efe25f8595b1a859dcd76d7c2649c6951c 100644 --- a/sage_validation/settings.py +++ b/sage_validation/settings.py @@ -18,7 +18,7 @@ BASE_DIR = Path(__file__).resolve().parent # See https://docs.djangoproject.com/en/5.1/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = os.getenv("SECRET_KEY") +SECRET_KEY = os.getenv("SECRET_KEY", "test-secret-key") # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True diff --git a/sage_validation/templates/index.html b/sage_validation/templates/index.html index 7d7376ea866409f6da728b1c6a343fa98b8fb7a4..72db6b8b5598643485a9e64dcfd7d9b10aa1f65b 100644 --- a/sage_validation/templates/index.html +++ b/sage_validation/templates/index.html @@ -8,7 +8,7 @@ <h1 class="text-5xl md:text-6xl font-bold mb-12 text-blue-900">Welcome to Sage Validation</h1> <p class="text-xl md:text-2xl mb-16 text-gray-700">Click the button below to upload your file for validation.</p> - <a href="{% url "upload-file" %}" + <a href="{% url "upload-page" %}" class="inline-flex py-4 px-16 bg-blue-600 text-white text-lg md:text-xl font-bold rounded-full shadow-lg transition-transform transform hover:scale-105 focus:outline-none focus:ring-4 focus:ring-blue-300 focus:ring-opacity-50"> Upload File </a> diff --git a/test/__init__.py b/test/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/test/conftest.py b/test/conftest.py new file mode 100644 index 0000000000000000000000000000000000000000..f32a5947d4e438df30201bec9d863bbd9e486ae5 --- /dev/null +++ b/test/conftest.py @@ -0,0 +1,147 @@ +"""Fixtures for the sage_validation tests.""" +from unittest.mock import MagicMock + +import pytest +from django.core.files.uploadedfile import SimpleUploadedFile +from faker import Faker +from rest_framework.test import APIClient + +from sage_validation.file_validator.models import MeoCostCentres, MeoValidSuppliers, XxData + + +@pytest.fixture +def sample_input_file() -> SimpleUploadedFile: + """Create a sample valid CSV file for testing.""" + csv_headers_list = [ + "AccountNumber", + "CBAccountNumber", + "DaysDiscountValid", + "DiscountValue", + "DiscountPercentage", + "DueDate", + "GoodsValueInAccountCurrency", + "PurControlValueInBaseCurrency", + "DocumentToBaseCurrencyRate", + "DocumentToAccountCurrencyRate", + "PostedDate", + "QueryCode", + "TransactionReference", + "SecondReference", + "Source", + "SYSTraderTranType", + "TransactionDate", + "UniqueReferenceNumber", + "UserNumber", + "TaxValue", + "SYSTraderGenerationReasonType", + "GoodsValueInBaseCurrency", + + # NominalAnalysis repeating columns (Example: /1 for first occurrence) + "NominalAnalysisTransactionValue/1", + "NominalAnalysisNominalAccountNumber/1", + "NominalAnalysisNominalCostCentre/1", + "NominalAnalysisNominalDepartment/1", + "NominalAnalysisNominalAnalysisNarrative/1", + "NominalAnalysisTransactionAnalysisCode/1", + + # TaxAnalysis repeating columns (Example: /1 for first occurrence) + "TaxAnalysisTaxRate/1", + "TaxAnalysisGoodsValueBeforeDiscount/1", + "TaxAnalysisDiscountValue/1", + "TaxAnalysisDiscountPercentage/1", + "TaxAnalysisTaxOnGoodsValue/1", + ] + csv_headers = ",".join(csv_headers_list) + + csv_content_list = [ + "12345", # AccountNumber + "54321", # CBAccountNumber + "30", # DaysDiscountValid + "10.5", # DiscountValue + "5.0", # DiscountPercentage + "2024-02-10", # DueDate + "1000", # GoodsValueInAccountCurrency + "950", # PurControlValueInBaseCurrency + "1.2", # DocumentToBaseCurrencyRate + "1.1", # DocumentToAccountCurrencyRate + "2024-02-05", # PostedDate + "Q1", # QueryCode + "TRX123", # TransactionReference + "SREF123", # SecondReference + "80", # Source + "4", # SYSTraderTranType + "2024-02-01", # TransactionDate + "UR123", # UniqueReferenceNumber + "42", # UserNumber + "10", # TaxValue + "1000", # SYSTraderGenerationReasonType + "1200", # GoodsValueInBaseCurrency + + # NominalAnalysis repeating values (Example: /1) + "500.75", # NominalAnalysisTransactionValue/1 + "ACC100", # NominalAnalysisNominalAccountNumber/1 + "CC100", # NominalAnalysisNominalCostCentre/1 + "DEP200", # NominalAnalysisNominalDepartment/1 + "Sample Narrative", # NominalAnalysisNominalAnalysisNarrative/1 + "TAC100", # NominalAnalysisTransactionAnalysisCode/1 + + # TaxAnalysis repeating values (Example: /1) + "20.5", # TaxAnalysisTaxRate/1 + "900", # TaxAnalysisGoodsValueBeforeDiscount/1 + "30", # TaxAnalysisDiscountValue/1 + "3.5", # TaxAnalysisDiscountPercentage/1 + "180", # TaxAnalysisTaxOnGoodsValue/1 + ] + csv_content = ",".join(csv_content_list) + + return SimpleUploadedFile("test.csv", f"{csv_headers}\n{csv_content}".encode(), content_type="text/csv") + +@pytest.fixture +def mock_meo_database(mocker: MagicMock)-> None: + """Mock the meo database since it's read-only.""" + fake = Faker() + + # Mock MeoValidSuppliers + supplier_mock = MagicMock() + supplier_mock.all.return_value = [ + MeoValidSuppliers( + supplier_account_number=str(fake.random_int(min=10000, max=99999)), supplier_account_name=fake.company() + ), + MeoValidSuppliers(supplier_account_number="12345", supplier_account_name="Sample Narrative") + ] + mocker.patch("sage_validation.file_validator.models.MeoValidSuppliers.objects.using", return_value=supplier_mock) + + # Mock MeoCostCentres + cost_centre_mock = MagicMock() + cost_centre_mock.all.return_value = [ + MeoCostCentres(cc="CC100", cc_type="Project", cc_name="CostCentreName", id=1), + MeoCostCentres(cc="CC200", cc_type="Overhead", cc_name="OverheadName", id=2), + MeoCostCentres(cc="CC300", cc_type="Overhead", cc_name="DepartmentName", id=3) + ] + mocker.patch("sage_validation.file_validator.models.MeoCostCentres.objects.using", return_value=cost_centre_mock) + + # Mock XxData + xx_data_mock = MagicMock() + xx_data_mock.all.return_value = [ + XxData(xx_value="N100", project="ProjectCode", overhead="OverheadCode", description=fake.sentence()), + XxData(xx_value="N200", project="ProjectCode", overhead="OverheadCode", description=fake.sentence()) + ] + mocker.patch("sage_validation.file_validator.models.XxData.objects.using", return_value=xx_data_mock) + + # Mock MeoValidSageAccounts + sage_account_mock = MagicMock() + sage_account_mock.filter.return_value.exists.return_value = True + mocker.patch( + "sage_validation.file_validator.models.MeoValidSageAccounts.objects.using", return_value=sage_account_mock + ) + + # Mock MeoNominal + nominal_mock = MagicMock() + nominal_mock.filter.return_value.exists.return_value = True + mocker.patch("sage_validation.file_validator.models.MeoNominal.objects.using", return_value=nominal_mock) + + +@pytest.fixture +def api_client() -> APIClient: + """Fixture to return Django API test client.""" + return APIClient() diff --git a/test/settings.py b/test/settings.py new file mode 100644 index 0000000000000000000000000000000000000000..225f265277cb055861640557b8c6992e3ee8ca79 --- /dev/null +++ b/test/settings.py @@ -0,0 +1,7 @@ +"""Settings for running tests.""" + +from sage_validation.settings import * # noqa: F403 + +DATABASES = { + "default": DATABASES["default"], # noqa: F405 +} diff --git a/test/test_file_validator/__init__.py b/test/test_file_validator/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/test/test_file_validator/test_file_validator_endpoints.py b/test/test_file_validator/test_file_validator_endpoints.py new file mode 100644 index 0000000000000000000000000000000000000000..3193a62b63d3e73eb1bd9d5f46127f3adec51a49 --- /dev/null +++ b/test/test_file_validator/test_file_validator_endpoints.py @@ -0,0 +1,52 @@ +from unittest.mock import MagicMock + +import pytest +from django.core.files.uploadedfile import SimpleUploadedFile +from django.urls.base import reverse +from rest_framework.test import APIClient + +UPLOAD_FILE_URL = reverse("upload-file") + + +@pytest.mark.django_db +def test_csv_upload_valid( + api_client: APIClient, sample_input_file: SimpleUploadedFile, mock_meo_database: MagicMock +) -> None: + """Test that a valid CSV upload succeeds.""" + response = api_client.post(UPLOAD_FILE_URL, {"file": sample_input_file}, format="multipart") + + assert response.status_code == 200 + assert response.json()["status"] == "success" + assert "download_url" in response.json() + + +@pytest.mark.django_db +def test_csv_upload_invalid_extension(api_client: APIClient) -> None: + """Test that uploading a non-CSV file fails.""" + bad_file = SimpleUploadedFile("test.txt", b"Invalid content", content_type="text/plain") + + response = api_client.post(UPLOAD_FILE_URL, {"file": bad_file}, format="multipart") + + assert response.status_code == 400 + assert response.json()["status"] == "error" + assert "errors" in response.json() + + +@pytest.mark.django_db +def test_csv_export_with_data(api_client: APIClient) -> None: + """Test exporting a processed CSV file.""" + url = reverse("export-file") + + # Simulate session data + session = api_client.session + session["validated_csv"] = [ + {"AccountNumber": "12345", "TransactionDate": "01/03/2024", "NominalAnalysisNominalAccountNumber/1": "N100"} + ] + session.save() + + response = api_client.get(url) + + assert response.status_code == 200 + assert response["Content-Disposition"] == 'attachment; filename="updated_file.csv"' + assert b"AccountNumber,TransactionDate,NominalAnalysisNominalAccountNumber/1" in response.content + assert b"12345,01/03/2024,N100" in response.content diff --git a/test/test_file_validator/test_forms.py b/test/test_file_validator/test_forms.py new file mode 100644 index 0000000000000000000000000000000000000000..b5df1089c1430e9a898b62cd8449ef849415f422 --- /dev/null +++ b/test/test_file_validator/test_forms.py @@ -0,0 +1,94 @@ +"""Tests for the file_validator forms.""" +import csv +import io +from unittest.mock import MagicMock + +from django.core.files.uploadedfile import SimpleUploadedFile + +from sage_validation.file_validator.forms import CSVUploadForm + + +def create_modified_csv(sample_file: SimpleUploadedFile, modifications: dict[str, str]) -> SimpleUploadedFile: + """ + Modify specific fields in the first row of a CSV file and return a new SimpleUploadedFile. + + Args: + sample_file (SimpleUploadedFile): The original CSV file. + modifications (dict): Dictionary of column names to modified values. + + Returns: + SimpleUploadedFile: The modified CSV file. + """ + csv_content = sample_file.read().decode("utf-8").splitlines() + reader = csv.DictReader(csv_content) + rows = list(reader) + + for key, value in modifications.items(): + rows[0][key] = value + + output = io.StringIO() + writer = csv.DictWriter(output, fieldnames=reader.fieldnames or []) + writer.writeheader() + writer.writerows(rows) + + return SimpleUploadedFile("test_modified.csv", output.getvalue().encode("utf-8"), content_type="text/csv") + + +def test_valid_csv_upload(sample_input_file: SimpleUploadedFile, mock_meo_database: MagicMock) -> None: + """Test CSV upload with valid data.""" + form = CSVUploadForm(files={"file": sample_input_file}) + assert form.is_valid(), f"Form errors: {form.errors}" + + +def test_invalid_file_extension() -> None: + """Test form rejects non-CSV files.""" + bad_file = SimpleUploadedFile("test.txt", b"Some text content", content_type="text/plain") + form = CSVUploadForm(files={"file": bad_file}) + assert not form.is_valid() + assert "File must be in CSV format." in form.errors["file"] + + +def test_missing_required_columns() -> None: + """Test form rejects CSV missing required headers.""" + invalid_csv = SimpleUploadedFile("test.csv", b"AccountNumber,CBAccountNumber\n12345,54321", content_type="text/csv") + form = CSVUploadForm(files={"file": invalid_csv}) + assert not form.is_valid() + assert "Missing required columns" in str(form.errors) + + +def test_source_and_trader_type_validation(sample_input_file: SimpleUploadedFile, mock_meo_database: MagicMock) -> None: + """Test validation for Source and SYSTraderTranType columns.""" + modified_file = create_modified_csv(sample_input_file, {"Source": "90", "SYSTraderTranType": "5"}) + form = CSVUploadForm(files={"file": modified_file}) + assert not form.is_valid() + assert "Row 1: 'Source' must be 80" in form.errors["file"][0] + assert "Row 1: 'SYSTraderTranType' must be 4" in form.errors["file"][1] + + +def test_validate_nominal_analysis_account(sample_input_file: SimpleUploadedFile, mock_meo_database: MagicMock) -> None: + """Test validation for nominal analysis account.""" + modified_file = create_modified_csv(sample_input_file, + {"NominalAnalysisNominalAnalysisNarrative/1": "Invalid Name"}) + form = CSVUploadForm(files={"file": modified_file}) + assert not form.is_valid() + assert form.errors["file"][0] == ( + "Row 1: 'AccountNumber' must match 'Sample Narrative' in 'NominalAnalysisNominalAnalysisNarrative/1'" + ", but found 'Invalid Name'.") + + +def test_validate_nc_cc_dep_combination_against_meo_sage_account( + sample_input_file: SimpleUploadedFile, mock_meo_database: MagicMock +) -> None: + """Test validation for nominal analysis fields against MEO valid Sage accounts.""" + modified_file = create_modified_csv( + sample_input_file, + { + "NominalAnalysisNominalCostCentre/1": "Invalid_CC", + "NominalAnalysisNominalAccountNumber/1": "Invalid_Account" + }) + + form = CSVUploadForm(files={"file": modified_file}) + assert not form.is_valid() + assert ("Row 1: 'NominalAnalysisNominalCostCentre/1' (Invalid_CC) is not a valid cost centre." + in str(form.errors["file"][0])) + diff --git a/tox.ini b/tox.ini index 91d0161e6590165ecf51bec47ab84ff68718aacd..9380288f26e73d609400dbc834530157acacb636 100644 --- a/tox.ini +++ b/tox.ini @@ -5,6 +5,13 @@ envlist = py311 deps = mypy ruff + -r requirements.txt commands = ruff check . mypy . + coverage erase + coverage run --source sage_validation -m pytest + coverage report --fail-under=90 + coverage xml + coverage html +