Skip to content
Snippets Groups Projects
Commit eedc72df authored by Neda Moeini's avatar Neda Moeini
Browse files

Merge branch 'feature/unit-test-for-validations' into 'develop' 

Feature/unit test for validations

See merge request !2
parents 77a93bcd cdbcc75a
No related branches found
No related tags found
1 merge request!2Feature/unit test for validations
Hide whitespace changes
Inline Side-by-side
Showing
with 369 additions and 81 deletions
pyproject.toml
+ 6
− 0
View file @ eedc72df
......@@ -37,4 +37,10 @@ line-length = 120
[tool.ruff.lint.flake8-tidy-imports]
ban-relative-imports = "all"
[tool.ruff.lint.per-file-ignores]
"test/*" = ["ARG001", "D", "S101", "PLR2004", "PLR0917", "PLR0914", "PLC0415", "PLC2701"]
[tool.pytest.ini_options]
DJANGO_SETTINGS_MODULE = "test.settings"
django_find_project = false
python_files = ["tests.py", "test_*.py", "*_tests.py"]
\ No newline at end of file
sage_validation/file_validator/templates/upload.html
+ 15
− 43
View file @ eedc72df
......@@ -40,21 +40,18 @@
</div>
</div>
<script>
const form = document.getElementById('uploadForm');
const fileInput = document.getElementById('fileInput');
const errorSection = document.getElementById('errorSection');
const errorList = document.getElementById('errorList');
const successSection = document.getElementById('successSection');
const successMessage = document.getElementById('successMessage');
const downloadSection = document.getElementById('downloadSection');
const downloadLink = document.getElementById('downloadLink');
form.addEventListener('submit', async function (e) {
document.getElementById('uploadForm').addEventListener('submit', async function (e) {
e.preventDefault();
// Clear previous messages
const fileInput = document.getElementById('fileInput');
const errorSection = document.getElementById('errorSection');
const errorList = document.getElementById('errorList');
const successSection = document.getElementById('successSection');
const successMessage = document.getElementById('successMessage');
const downloadSection = document.getElementById('downloadSection');
const downloadLink = document.getElementById('downloadLink');
errorList.innerHTML = '';
successMessage.innerHTML = '';
errorSection.classList.add('hidden');
......@@ -65,7 +62,7 @@
formData.append('file', fileInput.files[0]);
try {
const response = await fetch('', {
const response = await fetch("{% url 'upload-file' %}", {
method: 'POST',
body: formData,
headers: {
......@@ -81,38 +78,13 @@
downloadLink.href = result.download_url;
downloadSection.classList.remove('hidden');
} else if (response.status === 400 && result.status === 'error') {
errorList.innerHTML = '';
if (Array.isArray(result.errors)) {
result.errors.forEach(errorObj => {
if (typeof errorObj === 'string') {
const li = document.createElement('li');
li.textContent = errorObj;
errorList.appendChild(li);
} else {
for (const [field, messages] of Object.entries(errorObj)) {
messages.forEach(message => {
const li = document.createElement('li');
li.textContent = `${field}: ${message}`;
errorList.appendChild(li);
});
}
}
for (const [field, messages] of Object.entries(result.errors)) {
messages.forEach(message => {
const li = document.createElement('li');
li.textContent = `${field}: ${message}`;
errorList.appendChild(li);
});
} else if (typeof result.errors === 'object') {
for (const [field, messages] of Object.entries(result.errors)) {
messages.forEach(message => {
const li = document.createElement('li');
li.textContent = `${field}: ${message}`;
errorList.appendChild(li);
});
}
} else {
const li = document.createElement('li');
li.textContent = result.errors;
errorList.appendChild(li);
}
errorSection.classList.remove('hidden');
}
} catch (error) {
......
sage_validation/file_validator/urls.py
+ 4
− 3
View file @ eedc72df
......@@ -2,9 +2,10 @@
from django.urls import path
from sage_validation.file_validator.views import CSVExportView, CSVUploadView
from sage_validation.file_validator.views import CSVExportAPIView, CSVUploadAPIView, upload_page_view
urlpatterns = [
path("upload/", CSVUploadView.as_view(), name="upload-file"),
path("export/", CSVExportView.as_view(), name="export-file"),
path("upload-page/", upload_page_view, name="upload-page"),
path("api/upload/", CSVUploadAPIView.as_view(), name="upload-file"),
path("api/export/", CSVExportAPIView.as_view(), name="export-file"),
]
sage_validation/file_validator/views.py
+ 29
− 33
View file @ eedc72df
"""Views for the file_validator app."""
import csv
import io
from typing import Any
from django.http import HttpRequest, HttpResponse, JsonResponse
from django.http import HttpRequest, HttpResponse
from django.shortcuts import render
from django.urls import reverse_lazy
from django.utils import timezone
from django.views.generic.base import View
from django.views.generic.edit import FormView
from rest_framework import status
from rest_framework.request import Request
from rest_framework.response import Response
from rest_framework.views import APIView
from sage_validation.file_validator.forms import CSVUploadForm
from sage_validation.file_validator.models import MeoCostCentres, XxData
......@@ -19,46 +20,41 @@ def index_view(request: HttpRequest) -> HttpResponse:
return render(request, "index.html")
class CSVUploadView(FormView):
"""View for uploading a CSV file."""
def upload_page_view(request: HttpRequest) -> HttpResponse:
"""Render the file upload page."""
return render(request, "upload.html")
template_name = "upload.html"
form_class = CSVUploadForm
success_url = reverse_lazy("upload-file")
def get_context_data(self, **kwargs: dict[str, Any]) -> dict[str, Any]:
"""Render the form with no error message on GET request."""
context = super().get_context_data(**kwargs)
context["error"] = None
context["message"] = None
return context
class CSVUploadAPIView(APIView):
"""API view for uploading a CSV file."""
def post(self, request: Request) -> Response:
"""Handle CSV upload and validation."""
form = CSVUploadForm(data=request.data, files=request.FILES)
if not form.is_valid():
return Response({"status": "error", "errors": form.errors}, status=status.HTTP_400_BAD_REQUEST)
def form_valid(self, form: CSVUploadForm) -> JsonResponse:
"""Handle the CSV validation, store valid data, and prepare for export."""
csv_file = form.cleaned_data["file"]
csv_file.seek(0)
decoded_file = csv_file.read().decode("utf-8").strip()
if not decoded_file:
return JsonResponse({"status": "error", "message": "Uploaded file is empty."}, status=400)
return Response({"status": "error", "message": "Uploaded file is empty."},
status=status.HTTP_400_BAD_REQUEST)
reader = csv.DictReader(io.StringIO(decoded_file))
csv_data: list[dict[str, str]] = list(reader)
updated_data = self.update_fields(csv_data)
request.session["validated_csv"] = updated_data
request.session.modified = True
self.request.session["validated_csv"] = updated_data
self.request.session.modified = True
return JsonResponse({
return Response({
"status": "success",
"message": "File successfully uploaded and processed.",
"download_url": reverse_lazy("export-file")
})
def form_invalid(self, form: CSVUploadForm) -> JsonResponse:
"""Handle the form when it is invalid."""
return JsonResponse({"status": "error", "errors": form.errors}, status=400)
}, status=status.HTTP_200_OK)
@staticmethod
def update_fields(csv_data: list[dict[str, str]]) -> list[dict[str, str]]:
......@@ -91,21 +87,21 @@ class CSVUploadView(FormView):
row[f"NominalAnalysisNominalAccountNumber/{repeat}"] = (
xx_data[0] if cc_type == "Project" else xx_data[1]
)
repeat += 1
return csv_data
class CSVExportView(View):
"""View for exporting the updated CSV file."""
class CSVExportAPIView(APIView):
"""API view for exporting the updated CSV file."""
def get(self, request: HttpRequest) -> HttpResponse:
"""Generate a downloadable CSV file with updated values."""
def get(self, request: Request) -> Response:
"""Return processed CSV as a downloadable response."""
csv_data: list[dict[str, str]] = request.session.get("validated_csv", [])
if not csv_data:
return HttpResponse("No data available for export.", status=400)
return Response({"status": "error", "message": "No data available for export."},
status=status.HTTP_400_BAD_REQUEST)
response = HttpResponse(content_type="text/csv")
response["Content-Disposition"] = 'attachment; filename="updated_file.csv"'
......
sage_validation/settings.py
+ 1
− 1
View file @ eedc72df
......@@ -18,7 +18,7 @@ BASE_DIR = Path(__file__).resolve().parent
# See https://docs.djangoproject.com/en/5.1/howto/deployment/checklist/
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = os.getenv("SECRET_KEY")
SECRET_KEY = os.getenv("SECRET_KEY", "test-secret-key")
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True
......
sage_validation/templates/index.html
+ 1
− 1
View file @ eedc72df
......@@ -8,7 +8,7 @@
<h1 class="text-5xl md:text-6xl font-bold mb-12 text-blue-900">Welcome to Sage Validation</h1>
<p class="text-xl md:text-2xl mb-16 text-gray-700">Click the button below to upload your file for validation.</p>
<a href="{% url "upload-file" %}"
<a href="{% url "upload-page" %}"
class="inline-flex py-4 px-16 bg-blue-600 text-white text-lg md:text-xl font-bold rounded-full shadow-lg transition-transform transform hover:scale-105 focus:outline-none focus:ring-4 focus:ring-blue-300 focus:ring-opacity-50">
Upload File
</a>
......
test/__init__.py 0 → 100644
+ 0
− 0
View file @ eedc72df
test/conftest.py 0 → 100644
+ 147
− 0
View file @ eedc72df
"""Fixtures for the sage_validation tests."""
from unittest.mock import MagicMock
import pytest
from django.core.files.uploadedfile import SimpleUploadedFile
from faker import Faker
from rest_framework.test import APIClient
from sage_validation.file_validator.models import MeoCostCentres, MeoValidSuppliers, XxData
@pytest.fixture
def sample_input_file() -> SimpleUploadedFile:
"""Create a sample valid CSV file for testing."""
csv_headers_list = [
"AccountNumber",
"CBAccountNumber",
"DaysDiscountValid",
"DiscountValue",
"DiscountPercentage",
"DueDate",
"GoodsValueInAccountCurrency",
"PurControlValueInBaseCurrency",
"DocumentToBaseCurrencyRate",
"DocumentToAccountCurrencyRate",
"PostedDate",
"QueryCode",
"TransactionReference",
"SecondReference",
"Source",
"SYSTraderTranType",
"TransactionDate",
"UniqueReferenceNumber",
"UserNumber",
"TaxValue",
"SYSTraderGenerationReasonType",
"GoodsValueInBaseCurrency",
# NominalAnalysis repeating columns (Example: /1 for first occurrence)
"NominalAnalysisTransactionValue/1",
"NominalAnalysisNominalAccountNumber/1",
"NominalAnalysisNominalCostCentre/1",
"NominalAnalysisNominalDepartment/1",
"NominalAnalysisNominalAnalysisNarrative/1",
"NominalAnalysisTransactionAnalysisCode/1",
# TaxAnalysis repeating columns (Example: /1 for first occurrence)
"TaxAnalysisTaxRate/1",
"TaxAnalysisGoodsValueBeforeDiscount/1",
"TaxAnalysisDiscountValue/1",
"TaxAnalysisDiscountPercentage/1",
"TaxAnalysisTaxOnGoodsValue/1",
]
csv_headers = ",".join(csv_headers_list)
csv_content_list = [
"12345", # AccountNumber
"54321", # CBAccountNumber
"30", # DaysDiscountValid
"10.5", # DiscountValue
"5.0", # DiscountPercentage
"2024-02-10", # DueDate
"1000", # GoodsValueInAccountCurrency
"950", # PurControlValueInBaseCurrency
"1.2", # DocumentToBaseCurrencyRate
"1.1", # DocumentToAccountCurrencyRate
"2024-02-05", # PostedDate
"Q1", # QueryCode
"TRX123", # TransactionReference
"SREF123", # SecondReference
"80", # Source
"4", # SYSTraderTranType
"2024-02-01", # TransactionDate
"UR123", # UniqueReferenceNumber
"42", # UserNumber
"10", # TaxValue
"1000", # SYSTraderGenerationReasonType
"1200", # GoodsValueInBaseCurrency
# NominalAnalysis repeating values (Example: /1)
"500.75", # NominalAnalysisTransactionValue/1
"ACC100", # NominalAnalysisNominalAccountNumber/1
"CC100", # NominalAnalysisNominalCostCentre/1
"DEP200", # NominalAnalysisNominalDepartment/1
"Sample Narrative", # NominalAnalysisNominalAnalysisNarrative/1
"TAC100", # NominalAnalysisTransactionAnalysisCode/1
# TaxAnalysis repeating values (Example: /1)
"20.5", # TaxAnalysisTaxRate/1
"900", # TaxAnalysisGoodsValueBeforeDiscount/1
"30", # TaxAnalysisDiscountValue/1
"3.5", # TaxAnalysisDiscountPercentage/1
"180", # TaxAnalysisTaxOnGoodsValue/1
]
csv_content = ",".join(csv_content_list)
return SimpleUploadedFile("test.csv", f"{csv_headers}\n{csv_content}".encode(), content_type="text/csv")
@pytest.fixture
def mock_meo_database(mocker: MagicMock)-> None:
"""Mock the meo database since it's read-only."""
fake = Faker()
# Mock MeoValidSuppliers
supplier_mock = MagicMock()
supplier_mock.all.return_value = [
MeoValidSuppliers(
supplier_account_number=str(fake.random_int(min=10000, max=99999)), supplier_account_name=fake.company()
),
MeoValidSuppliers(supplier_account_number="12345", supplier_account_name="Sample Narrative")
]
mocker.patch("sage_validation.file_validator.models.MeoValidSuppliers.objects.using", return_value=supplier_mock)
# Mock MeoCostCentres
cost_centre_mock = MagicMock()
cost_centre_mock.all.return_value = [
MeoCostCentres(cc="CC100", cc_type="Project", cc_name="CostCentreName", id=1),
MeoCostCentres(cc="CC200", cc_type="Overhead", cc_name="OverheadName", id=2),
MeoCostCentres(cc="CC300", cc_type="Overhead", cc_name="DepartmentName", id=3)
]
mocker.patch("sage_validation.file_validator.models.MeoCostCentres.objects.using", return_value=cost_centre_mock)
# Mock XxData
xx_data_mock = MagicMock()
xx_data_mock.all.return_value = [
XxData(xx_value="N100", project="ProjectCode", overhead="OverheadCode", description=fake.sentence()),
XxData(xx_value="N200", project="ProjectCode", overhead="OverheadCode", description=fake.sentence())
]
mocker.patch("sage_validation.file_validator.models.XxData.objects.using", return_value=xx_data_mock)
# Mock MeoValidSageAccounts
sage_account_mock = MagicMock()
sage_account_mock.filter.return_value.exists.return_value = True
mocker.patch(
"sage_validation.file_validator.models.MeoValidSageAccounts.objects.using", return_value=sage_account_mock
)
# Mock MeoNominal
nominal_mock = MagicMock()
nominal_mock.filter.return_value.exists.return_value = True
mocker.patch("sage_validation.file_validator.models.MeoNominal.objects.using", return_value=nominal_mock)
@pytest.fixture
def api_client() -> APIClient:
"""Fixture to return Django API test client."""
return APIClient()
test/settings.py 0 → 100644
+ 7
− 0
View file @ eedc72df
"""Settings for running tests."""
from sage_validation.settings import * # noqa: F403
DATABASES = {
"default": DATABASES["default"], # noqa: F405
}
test/test_file_validator/__init__.py 0 → 100644
+ 0
− 0
View file @ eedc72df
test/test_file_validator/test_file_validator_endpoints.py 0 → 100644
+ 52
− 0
View file @ eedc72df
from unittest.mock import MagicMock
import pytest
from django.core.files.uploadedfile import SimpleUploadedFile
from django.urls.base import reverse
from rest_framework.test import APIClient
UPLOAD_FILE_URL = reverse("upload-file")
@pytest.mark.django_db
def test_csv_upload_valid(
api_client: APIClient, sample_input_file: SimpleUploadedFile, mock_meo_database: MagicMock
) -> None:
"""Test that a valid CSV upload succeeds."""
response = api_client.post(UPLOAD_FILE_URL, {"file": sample_input_file}, format="multipart")
assert response.status_code == 200
assert response.json()["status"] == "success"
assert "download_url" in response.json()
@pytest.mark.django_db
def test_csv_upload_invalid_extension(api_client: APIClient) -> None:
"""Test that uploading a non-CSV file fails."""
bad_file = SimpleUploadedFile("test.txt", b"Invalid content", content_type="text/plain")
response = api_client.post(UPLOAD_FILE_URL, {"file": bad_file}, format="multipart")
assert response.status_code == 400
assert response.json()["status"] == "error"
assert "errors" in response.json()
@pytest.mark.django_db
def test_csv_export_with_data(api_client: APIClient) -> None:
"""Test exporting a processed CSV file."""
url = reverse("export-file")
# Simulate session data
session = api_client.session
session["validated_csv"] = [
{"AccountNumber": "12345", "TransactionDate": "01/03/2024", "NominalAnalysisNominalAccountNumber/1": "N100"}
]
session.save()
response = api_client.get(url)
assert response.status_code == 200
assert response["Content-Disposition"] == 'attachment; filename="updated_file.csv"'
assert b"AccountNumber,TransactionDate,NominalAnalysisNominalAccountNumber/1" in response.content
assert b"12345,01/03/2024,N100" in response.content
test/test_file_validator/test_forms.py 0 → 100644
+ 94
− 0
View file @ eedc72df
"""Tests for the file_validator forms."""
import csv
import io
from unittest.mock import MagicMock
from django.core.files.uploadedfile import SimpleUploadedFile
from sage_validation.file_validator.forms import CSVUploadForm
def create_modified_csv(sample_file: SimpleUploadedFile, modifications: dict[str, str]) -> SimpleUploadedFile:
"""
Modify specific fields in the first row of a CSV file and return a new SimpleUploadedFile.
Args:
sample_file (SimpleUploadedFile): The original CSV file.
modifications (dict): Dictionary of column names to modified values.
Returns:
SimpleUploadedFile: The modified CSV file.
"""
csv_content = sample_file.read().decode("utf-8").splitlines()
reader = csv.DictReader(csv_content)
rows = list(reader)
for key, value in modifications.items():
rows[0][key] = value
output = io.StringIO()
writer = csv.DictWriter(output, fieldnames=reader.fieldnames or [])
writer.writeheader()
writer.writerows(rows)
return SimpleUploadedFile("test_modified.csv", output.getvalue().encode("utf-8"), content_type="text/csv")
def test_valid_csv_upload(sample_input_file: SimpleUploadedFile, mock_meo_database: MagicMock) -> None:
"""Test CSV upload with valid data."""
form = CSVUploadForm(files={"file": sample_input_file})
assert form.is_valid(), f"Form errors: {form.errors}"
def test_invalid_file_extension() -> None:
"""Test form rejects non-CSV files."""
bad_file = SimpleUploadedFile("test.txt", b"Some text content", content_type="text/plain")
form = CSVUploadForm(files={"file": bad_file})
assert not form.is_valid()
assert "File must be in CSV format." in form.errors["file"]
def test_missing_required_columns() -> None:
"""Test form rejects CSV missing required headers."""
invalid_csv = SimpleUploadedFile("test.csv", b"AccountNumber,CBAccountNumber\n12345,54321", content_type="text/csv")
form = CSVUploadForm(files={"file": invalid_csv})
assert not form.is_valid()
assert "Missing required columns" in str(form.errors)
def test_source_and_trader_type_validation(sample_input_file: SimpleUploadedFile, mock_meo_database: MagicMock) -> None:
"""Test validation for Source and SYSTraderTranType columns."""
modified_file = create_modified_csv(sample_input_file, {"Source": "90", "SYSTraderTranType": "5"})
form = CSVUploadForm(files={"file": modified_file})
assert not form.is_valid()
assert "Row 1: 'Source' must be 80" in form.errors["file"][0]
assert "Row 1: 'SYSTraderTranType' must be 4" in form.errors["file"][1]
def test_validate_nominal_analysis_account(sample_input_file: SimpleUploadedFile, mock_meo_database: MagicMock) -> None:
"""Test validation for nominal analysis account."""
modified_file = create_modified_csv(sample_input_file,
{"NominalAnalysisNominalAnalysisNarrative/1": "Invalid Name"})
form = CSVUploadForm(files={"file": modified_file})
assert not form.is_valid()
assert form.errors["file"][0] == (
"Row 1: 'AccountNumber' must match 'Sample Narrative' in 'NominalAnalysisNominalAnalysisNarrative/1'"
", but found 'Invalid Name'.")
def test_validate_nc_cc_dep_combination_against_meo_sage_account(
sample_input_file: SimpleUploadedFile, mock_meo_database: MagicMock
) -> None:
"""Test validation for nominal analysis fields against MEO valid Sage accounts."""
modified_file = create_modified_csv(
sample_input_file,
{
"NominalAnalysisNominalCostCentre/1": "Invalid_CC",
"NominalAnalysisNominalAccountNumber/1": "Invalid_Account"
})
form = CSVUploadForm(files={"file": modified_file})
assert not form.is_valid()
assert ("Row 1: 'NominalAnalysisNominalCostCentre/1' (Invalid_CC) is not a valid cost centre."
in str(form.errors["file"][0]))
tox.ini
+ 7
− 0
View file @ eedc72df
......@@ -5,6 +5,13 @@ envlist = py311
deps =
mypy
ruff
-r requirements.txt
commands =
ruff check .
mypy .
coverage erase
coverage run --source sage_validation -m pytest
coverage report --fail-under=90
coverage xml
coverage html
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment