From cf96902b1f40b03684071d65a2f6834b6a15c42f Mon Sep 17 00:00:00 2001
From: Neda Moeini <neda.moeini@geant.org>
Date: Tue, 11 Feb 2025 14:59:40 +0100
Subject: [PATCH 1/9] Add pytest to the project and set up some basic fixtures
---
pytest.ini | 4 ++
requirements.txt | 4 ++
test/__init__.py | 0
test/conftest.py | 132 +++++++++++++++++++++++++++++++++++++++++++++++
test/settings.py | 5 ++
5 files changed, 145 insertions(+)
create mode 100644 pytest.ini
create mode 100644 test/__init__.py
create mode 100644 test/conftest.py
create mode 100644 test/settings.py
diff --git a/pytest.ini b/pytest.ini
new file mode 100644
index 0000000..8872a16
--- /dev/null
+++ b/pytest.ini
@@ -0,0 +1,4 @@
+[pytest]
+DJANGO_SETTINGS_MODULE = test.settings
+django_find_project = false
+python_files = tests.py test_*.py *_tests.py
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
index 0fb6979..af546b5 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -5,3 +5,7 @@ tox
sphinx
sphinx-autodoc-typehints
mssql-django
+pytest
+pytest-django
+pytest-mock
+faker
diff --git a/test/__init__.py b/test/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/test/conftest.py b/test/conftest.py
new file mode 100644
index 0000000..1d75fae
--- /dev/null
+++ b/test/conftest.py
@@ -0,0 +1,132 @@
+import pytest
+from unittest.mock import MagicMock
+from django.core.files.uploadedfile import SimpleUploadedFile
+from sage_validation.file_validator.models import MeoValidSuppliers, MeoCostCentres, XxData
+from faker import Faker
+
+
+@pytest.fixture
+def sample_input_file():
+ """Creates a sample valid CSV file for testing."""
+ csv_headers = ",".join([
+ "AccountNumber",
+ "CBAccountNumber",
+ "DaysDiscountValid",
+ "DiscountValue",
+ "DiscountPercentage",
+ "DueDate",
+ "GoodsValueInAccountCurrency",
+ "PurControlValueInBaseCurrency",
+ "DocumentToBaseCurrencyRate",
+ "DocumentToAccountCurrencyRate",
+ "PostedDate",
+ "QueryCode",
+ "TransactionReference",
+ "SecondReference",
+ "Source",
+ "SYSTraderTranType",
+ "TransactionDate",
+ "UniqueReferenceNumber",
+ "UserNumber",
+ "TaxValue",
+ "SYSTraderGenerationReasonType",
+ "GoodsValueInBaseCurrency",
+
+ # NominalAnalysis repeating columns (Example: /1 for first occurrence)
+ "NominalAnalysisTransactionValue/1",
+ "NominalAnalysisNominalAccountNumber/1",
+ "NominalAnalysisNominalCostCentre/1",
+ "NominalAnalysisNominalDepartment/1",
+ "NominalAnalysisNominalAnalysisNarrative/1",
+ "NominalAnalysisTransactionAnalysisCode/1",
+
+ # TaxAnalysis repeating columns (Example: /1 for first occurrence)
+ "TaxAnalysisTaxRate/1",
+ "TaxAnalysisGoodsValueBeforeDiscount/1",
+ "TaxAnalysisDiscountValue/1",
+ "TaxAnalysisDiscountPercentage/1",
+ "TaxAnalysisTaxOnGoodsValue/1",
+ ])
+
+ csv_content = ",".join([
+ "12345", # AccountNumber
+ "54321", # CBAccountNumber
+ "30", # DaysDiscountValid
+ "10.5", # DiscountValue
+ "5.0", # DiscountPercentage
+ "2024-02-10", # DueDate
+ "1000", # GoodsValueInAccountCurrency
+ "950", # PurControlValueInBaseCurrency
+ "1.2", # DocumentToBaseCurrencyRate
+ "1.1", # DocumentToAccountCurrencyRate
+ "2024-02-05", # PostedDate
+ "Q1", # QueryCode
+ "TRX123", # TransactionReference
+ "SREF123", # SecondReference
+ "80", # Source
+ "4", # SYSTraderTranType
+ "2024-02-01", # TransactionDate
+ "UR123", # UniqueReferenceNumber
+ "42", # UserNumber
+ "10", # TaxValue
+ "1000", # SYSTraderGenerationReasonType
+ "1200", # GoodsValueInBaseCurrency
+
+ # NominalAnalysis repeating values (Example: /1)
+ "500.75", # NominalAnalysisTransactionValue/1
+ "ACC100", # NominalAnalysisNominalAccountNumber/1
+ "CC100", # NominalAnalysisNominalCostCentre/1
+ "DEP200", # NominalAnalysisNominalDepartment/1
+ "Sample Narrative", # NominalAnalysisNominalAnalysisNarrative/1
+ "TAC100", # NominalAnalysisTransactionAnalysisCode/1
+
+ # TaxAnalysis repeating values (Example: /1)
+ "20.5", # TaxAnalysisTaxRate/1
+ "900", # TaxAnalysisGoodsValueBeforeDiscount/1
+ "30", # TaxAnalysisDiscountValue/1
+ "3.5", # TaxAnalysisDiscountPercentage/1
+ "180", # TaxAnalysisTaxOnGoodsValue/1
+ ])
+
+ return SimpleUploadedFile("test.csv", f"{csv_headers}\n{csv_content}".encode("utf-8"), content_type="text/csv")
+
+@pytest.fixture
+def mock_meo_database(mocker):
+ """Mock the meo database since it's read-only."""
+ fake = Faker()
+
+ # Mock MeoValidSuppliers
+ supplier_mock = MagicMock()
+ supplier_mock.all.return_value = [
+ MeoValidSuppliers(supplier_account_number=str(fake.random_int(min=10000, max=99999)), supplier_account_name=fake.company()),
+ MeoValidSuppliers(supplier_account_number="12345", supplier_account_name=fake.company())
+ ]
+ mocker.patch("sage_validation.file_validator.models.MeoValidSuppliers.objects.using", return_value=supplier_mock)
+
+ # Mock MeoCostCentres
+ cost_centre_mock = MagicMock()
+ cost_centre_mock.all.return_value = [
+ MeoCostCentres(cc="CC100", cc_type="Project", cc_name="CostCentreName", id=1),
+ MeoCostCentres(cc="CC200", cc_type="Overhead", cc_name="OverheadName", id=2),
+ MeoCostCentres(cc="CC300", cc_type="Overhead", cc_name="DepartmentName", id=3)
+ ]
+ mocker.patch("sage_validation.file_validator.models.MeoCostCentres.objects.using", return_value=cost_centre_mock)
+
+ # Mock XxData
+ xx_data_mock = MagicMock()
+ xx_data_mock.all.return_value = [
+ XxData(xx_value="N100", project="ProjectCode", overhead="OverheadCode", description=fake.sentence()),
+ XxData(xx_value="N200", project="ProjectCode", overhead="OverheadCode", description=fake.sentence())
+ ]
+ mocker.patch("sage_validation.file_validator.models.XxData.objects.using", return_value=xx_data_mock)
+
+ # Mock MeoValidSageAccounts
+ sage_account_mock = MagicMock()
+ sage_account_mock.filter.return_value.exists.return_value = True
+ mocker.patch("sage_validation.file_validator.models.MeoValidSageAccounts.objects.using", return_value=sage_account_mock)
+
+ # Mock MeoNominal
+ nominal_mock = MagicMock()
+ nominal_mock.filter.return_value.exists.return_value = True
+ mocker.patch("sage_validation.file_validator.models.MeoNominal.objects.using", return_value=nominal_mock)
+
diff --git a/test/settings.py b/test/settings.py
new file mode 100644
index 0000000..a5690fb
--- /dev/null
+++ b/test/settings.py
@@ -0,0 +1,5 @@
+from sage_validation.settings import * # noqa: F403, F401
+
+DATABASES = {
+ "default": DATABASES["default"],
+}
--
GitLab
From caf47cb1eaf1ecda50602fb1ffa09d0ea5df771d Mon Sep 17 00:00:00 2001
From: Neda Moeini <neda.moeini@geant.org>
Date: Thu, 27 Feb 2025 14:15:51 +0100
Subject: [PATCH 2/9] Add unittests for form validator
---
test/conftest.py | 2 +-
test/test_file_validator/__init__.py | 0
test/test_file_validator/test_forms.py | 52 ++++++++++++++++++++++++++
3 files changed, 53 insertions(+), 1 deletion(-)
create mode 100644 test/test_file_validator/__init__.py
create mode 100644 test/test_file_validator/test_forms.py
diff --git a/test/conftest.py b/test/conftest.py
index 1d75fae..dc7f3aa 100644
--- a/test/conftest.py
+++ b/test/conftest.py
@@ -99,7 +99,7 @@ def mock_meo_database(mocker):
supplier_mock = MagicMock()
supplier_mock.all.return_value = [
MeoValidSuppliers(supplier_account_number=str(fake.random_int(min=10000, max=99999)), supplier_account_name=fake.company()),
- MeoValidSuppliers(supplier_account_number="12345", supplier_account_name=fake.company())
+ MeoValidSuppliers(supplier_account_number="12345", supplier_account_name="Sample Narrative")
]
mocker.patch("sage_validation.file_validator.models.MeoValidSuppliers.objects.using", return_value=supplier_mock)
diff --git a/test/test_file_validator/__init__.py b/test/test_file_validator/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/test/test_file_validator/test_forms.py b/test/test_file_validator/test_forms.py
new file mode 100644
index 0000000..db0a25a
--- /dev/null
+++ b/test/test_file_validator/test_forms.py
@@ -0,0 +1,52 @@
+import csv
+import io
+
+from django.core.files.uploadedfile import SimpleUploadedFile
+
+from sage_validation.file_validator.forms import CSVUploadForm
+
+
+def test_valid_csv_upload(sample_input_file, mock_meo_database):
+ """Test CSV upload with valid data."""
+ form = CSVUploadForm(files={"file": sample_input_file})
+ assert form.is_valid(), f"Form errors: {form.errors}"
+
+
+def test_invalid_file_extension():
+ """Test form rejects non-CSV files."""
+ bad_file = SimpleUploadedFile("test.txt", b"Some text content", content_type="text/plain")
+ form = CSVUploadForm(files={"file": bad_file})
+ assert not form.is_valid()
+ assert "File must be in CSV format." in form.errors["file"]
+
+
+def test_missing_required_columns():
+ """Test form rejects CSV missing required headers."""
+ invalid_csv = SimpleUploadedFile("test.csv", b"AccountNumber,CBAccountNumber\n12345,54321", content_type="text/csv")
+ form = CSVUploadForm(files={"file": invalid_csv})
+ assert not form.is_valid()
+ assert "Missing required columns" in str(form.errors)
+
+
+def test_source_and_trader_type_validation(sample_input_file, mock_meo_database):
+ """Test validation for Source and SYSTraderTranType columns."""
+
+ csv_content = sample_input_file.read().decode("utf-8").splitlines()
+ reader = csv.DictReader(csv_content)
+ rows = list(reader)
+
+ # Modify the first row's "Source" and "SYSTraderTranType" values
+ rows[0]["Source"] = "90" # Change Source from 80 to 90
+ rows[0]["SYSTraderTranType"] = "5" # Change SYSTraderTranType from 4 to 5
+
+ # Rebuild the CSV with modified values
+ output = io.StringIO()
+ writer = csv.DictWriter(output, fieldnames=reader.fieldnames)
+ writer.writeheader()
+ writer.writerows(rows)
+ modified_file = SimpleUploadedFile("test_modified.csv", output.getvalue().encode("utf-8"), content_type="text/csv")
+
+ form = CSVUploadForm(files={"file": modified_file})
+ assert not form.is_valid(), "Form should be invalid due to incorrect Source and SYSTraderTranType values"
+ assert "Row 1: 'Source' must be 80" in form.errors["file"][0]
+ assert "Row 1: 'SYSTraderTranType' must be 4" in form.errors["file"][1]
--
GitLab
From 36a671c62bae9aef3170f7f22deb483196113b90 Mon Sep 17 00:00:00 2001
From: Neda Moeini <neda.moeini@geant.org>
Date: Thu, 27 Feb 2025 16:07:25 +0100
Subject: [PATCH 3/9] Test validate nc cc dep combination against meo sage
account
---
test/test_file_validator/test_forms.py | 44 +++++++++++++++++++++++++-
1 file changed, 43 insertions(+), 1 deletion(-)
diff --git a/test/test_file_validator/test_forms.py b/test/test_file_validator/test_forms.py
index db0a25a..583fbbe 100644
--- a/test/test_file_validator/test_forms.py
+++ b/test/test_file_validator/test_forms.py
@@ -47,6 +47,48 @@ def test_source_and_trader_type_validation(sample_input_file, mock_meo_database)
modified_file = SimpleUploadedFile("test_modified.csv", output.getvalue().encode("utf-8"), content_type="text/csv")
form = CSVUploadForm(files={"file": modified_file})
- assert not form.is_valid(), "Form should be invalid due to incorrect Source and SYSTraderTranType values"
+ assert not form.is_valid()
assert "Row 1: 'Source' must be 80" in form.errors["file"][0]
assert "Row 1: 'SYSTraderTranType' must be 4" in form.errors["file"][1]
+
+
+def test_validate_nominal_analysis_account(sample_input_file, mock_meo_database):
+ """Test validation for nominal analysis account."""
+
+ csv_content = sample_input_file.read().decode("utf-8").splitlines()
+ reader = csv.DictReader(csv_content)
+ rows = list(reader)
+ rows[0]["NominalAnalysisNominalAnalysisNarrative/1"] = "Invalid Name"
+
+ # Rebuild the CSV with modified values
+ output = io.StringIO()
+ writer = csv.DictWriter(output, fieldnames=reader.fieldnames)
+ writer.writeheader()
+ writer.writerows(rows)
+ modified_file = SimpleUploadedFile("test_modified.csv", output.getvalue().encode("utf-8"), content_type="text/csv")
+
+ form = CSVUploadForm(files={"file": modified_file})
+ assert not form.is_valid()
+ assert (f"Row 1: 'AccountNumber' must match 'Sample Narrative' in 'NominalAnalysisNominalAnalysisNarrative/1',"
+ f" but found 'Invalid Name'.") == form.errors["file"][0]
+
+
+def test_validate_nc_cc_dep_combination_against_meo_sage_account(sample_input_file, mock_meo_database):
+ """Test validation for nominal analysis fields against MEO valid Sage accounts."""
+ csv_content = sample_input_file.read().decode("utf-8").splitlines()
+ reader = csv.DictReader(csv_content)
+ rows = list(reader)
+
+ rows[0]["NominalAnalysisNominalCostCentre/1"] = "Invalid_CC"
+ rows[0]["NominalAnalysisNominalAccountNumber/1"] = "Invalid_Account"
+
+ output = io.StringIO()
+ writer = csv.DictWriter(output, fieldnames=reader.fieldnames)
+ writer.writeheader()
+ writer.writerows(rows)
+ modified_file = SimpleUploadedFile("test_modified.csv", output.getvalue().encode("utf-8"), content_type="text/csv")
+
+ form = CSVUploadForm(files={"file": modified_file})
+ assert not form.is_valid()
+ assert "Row 1: 'NominalAnalysisNominalCostCentre/1' (Invalid_CC) is not a valid cost centre." in str(form.errors["file"][0])
+
--
GitLab
From adb279d139cb8b1afc85e79e7fb838306da4d63d Mon Sep 17 00:00:00 2001
From: Neda Moeini <neda.moeini@geant.org>
Date: Thu, 27 Feb 2025 16:34:56 +0100
Subject: [PATCH 4/9] Make ruff happy
---
test/conftest.py | 29 +++++++++++++++++---------
test/settings.py | 6 ++++--
test/test_file_validator/test_forms.py | 11 +++++-----
3 files changed, 29 insertions(+), 17 deletions(-)
diff --git a/test/conftest.py b/test/conftest.py
index dc7f3aa..74bbc11 100644
--- a/test/conftest.py
+++ b/test/conftest.py
@@ -1,14 +1,17 @@
-import pytest
+"""Fixtures for the sage_validation tests."""
from unittest.mock import MagicMock
+
+import pytest
from django.core.files.uploadedfile import SimpleUploadedFile
-from sage_validation.file_validator.models import MeoValidSuppliers, MeoCostCentres, XxData
from faker import Faker
+from sage_validation.file_validator.models import MeoCostCentres, MeoValidSuppliers, XxData
+
@pytest.fixture
def sample_input_file():
- """Creates a sample valid CSV file for testing."""
- csv_headers = ",".join([
+ """Create a sample valid CSV file for testing."""
+ csv_headers_list = [
"AccountNumber",
"CBAccountNumber",
"DaysDiscountValid",
@@ -46,9 +49,10 @@ def sample_input_file():
"TaxAnalysisDiscountValue/1",
"TaxAnalysisDiscountPercentage/1",
"TaxAnalysisTaxOnGoodsValue/1",
- ])
+ ]
+ csv_headers = ",".join(csv_headers_list)
- csv_content = ",".join([
+ csv_content_list = [
"12345", # AccountNumber
"54321", # CBAccountNumber
"30", # DaysDiscountValid
@@ -86,9 +90,10 @@ def sample_input_file():
"30", # TaxAnalysisDiscountValue/1
"3.5", # TaxAnalysisDiscountPercentage/1
"180", # TaxAnalysisTaxOnGoodsValue/1
- ])
+ ]
+ csv_content = ",".join(csv_content_list)
- return SimpleUploadedFile("test.csv", f"{csv_headers}\n{csv_content}".encode("utf-8"), content_type="text/csv")
+ return SimpleUploadedFile("test.csv", f"{csv_headers}\n{csv_content}".encode(), content_type="text/csv")
@pytest.fixture
def mock_meo_database(mocker):
@@ -98,7 +103,9 @@ def mock_meo_database(mocker):
# Mock MeoValidSuppliers
supplier_mock = MagicMock()
supplier_mock.all.return_value = [
- MeoValidSuppliers(supplier_account_number=str(fake.random_int(min=10000, max=99999)), supplier_account_name=fake.company()),
+ MeoValidSuppliers(
+ supplier_account_number=str(fake.random_int(min=10000, max=99999)), supplier_account_name=fake.company()
+ ),
MeoValidSuppliers(supplier_account_number="12345", supplier_account_name="Sample Narrative")
]
mocker.patch("sage_validation.file_validator.models.MeoValidSuppliers.objects.using", return_value=supplier_mock)
@@ -123,7 +130,9 @@ def mock_meo_database(mocker):
# Mock MeoValidSageAccounts
sage_account_mock = MagicMock()
sage_account_mock.filter.return_value.exists.return_value = True
- mocker.patch("sage_validation.file_validator.models.MeoValidSageAccounts.objects.using", return_value=sage_account_mock)
+ mocker.patch(
+ "sage_validation.file_validator.models.MeoValidSageAccounts.objects.using", return_value=sage_account_mock
+ )
# Mock MeoNominal
nominal_mock = MagicMock()
diff --git a/test/settings.py b/test/settings.py
index a5690fb..225f265 100644
--- a/test/settings.py
+++ b/test/settings.py
@@ -1,5 +1,7 @@
-from sage_validation.settings import * # noqa: F403, F401
+"""Settings for running tests."""
+
+from sage_validation.settings import * # noqa: F403
DATABASES = {
- "default": DATABASES["default"],
+ "default": DATABASES["default"], # noqa: F405
}
diff --git a/test/test_file_validator/test_forms.py b/test/test_file_validator/test_forms.py
index 583fbbe..b169538 100644
--- a/test/test_file_validator/test_forms.py
+++ b/test/test_file_validator/test_forms.py
@@ -1,3 +1,4 @@
+"""Tests for the file_validator forms."""
import csv
import io
@@ -30,7 +31,6 @@ def test_missing_required_columns():
def test_source_and_trader_type_validation(sample_input_file, mock_meo_database):
"""Test validation for Source and SYSTraderTranType columns."""
-
csv_content = sample_input_file.read().decode("utf-8").splitlines()
reader = csv.DictReader(csv_content)
rows = list(reader)
@@ -54,7 +54,6 @@ def test_source_and_trader_type_validation(sample_input_file, mock_meo_database)
def test_validate_nominal_analysis_account(sample_input_file, mock_meo_database):
"""Test validation for nominal analysis account."""
-
csv_content = sample_input_file.read().decode("utf-8").splitlines()
reader = csv.DictReader(csv_content)
rows = list(reader)
@@ -69,8 +68,9 @@ def test_validate_nominal_analysis_account(sample_input_file, mock_meo_database)
form = CSVUploadForm(files={"file": modified_file})
assert not form.is_valid()
- assert (f"Row 1: 'AccountNumber' must match 'Sample Narrative' in 'NominalAnalysisNominalAnalysisNarrative/1',"
- f" but found 'Invalid Name'.") == form.errors["file"][0]
+ assert form.errors["file"][0] == (
+ "Row 1: 'AccountNumber' must match 'Sample Narrative' in 'NominalAnalysisNominalAnalysisNarrative/1'"
+ ", but found 'Invalid Name'.")
def test_validate_nc_cc_dep_combination_against_meo_sage_account(sample_input_file, mock_meo_database):
@@ -90,5 +90,6 @@ def test_validate_nc_cc_dep_combination_against_meo_sage_account(sample_input_fi
form = CSVUploadForm(files={"file": modified_file})
assert not form.is_valid()
- assert "Row 1: 'NominalAnalysisNominalCostCentre/1' (Invalid_CC) is not a valid cost centre." in str(form.errors["file"][0])
+ assert ("Row 1: 'NominalAnalysisNominalCostCentre/1' (Invalid_CC) is not a valid cost centre."
+ in str(form.errors["file"][0]))
--
GitLab
From 82236f1712bbf176556c219c11ed14ce9c65d1fe Mon Sep 17 00:00:00 2001
From: Neda Moeini <neda.moeini@geant.org>
Date: Thu, 27 Feb 2025 16:35:27 +0100
Subject: [PATCH 5/9] Move pytest configuration to pyproject.toml
---
pyproject.toml | 6 ++++++
pytest.ini | 4 ----
2 files changed, 6 insertions(+), 4 deletions(-)
delete mode 100644 pytest.ini
diff --git a/pyproject.toml b/pyproject.toml
index 491a227..981838d 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -37,4 +37,10 @@ line-length = 120
[tool.ruff.lint.flake8-tidy-imports]
ban-relative-imports = "all"
+[tool.ruff.lint.per-file-ignores]
+"test/*" = ["S101", "ARG001",]
+[tool.pytest.ini_options]
+DJANGO_SETTINGS_MODULE = "test.settings"
+django_find_project = false
+python_files = ["tests.py", "test_*.py", "*_tests.py"]
\ No newline at end of file
diff --git a/pytest.ini b/pytest.ini
deleted file mode 100644
index 8872a16..0000000
--- a/pytest.ini
+++ /dev/null
@@ -1,4 +0,0 @@
-[pytest]
-DJANGO_SETTINGS_MODULE = test.settings
-django_find_project = false
-python_files = tests.py test_*.py *_tests.py
\ No newline at end of file
--
GitLab
From e2033a5cc6318e60fee2be6877b3c2dce42970a7 Mon Sep 17 00:00:00 2001
From: Neda Moeini <neda.moeini@geant.org>
Date: Thu, 27 Feb 2025 16:35:49 +0100
Subject: [PATCH 6/9] Add coverage
---
requirements.txt | 1 +
tox.ini | 7 +++++++
2 files changed, 8 insertions(+)
diff --git a/requirements.txt b/requirements.txt
index af546b5..9732a52 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -9,3 +9,4 @@ pytest
pytest-django
pytest-mock
faker
+coverage
diff --git a/tox.ini b/tox.ini
index 91d0161..9380288 100644
--- a/tox.ini
+++ b/tox.ini
@@ -5,6 +5,13 @@ envlist = py311
deps =
mypy
ruff
+ -r requirements.txt
commands =
ruff check .
mypy .
+ coverage erase
+ coverage run --source sage_validation -m pytest
+ coverage report --fail-under=90
+ coverage xml
+ coverage html
+
--
GitLab
From e95043467108a410a198fddc8db0d5000f4b7f12 Mon Sep 17 00:00:00 2001
From: Neda Moeini <neda.moeini@geant.org>
Date: Fri, 28 Feb 2025 11:06:54 +0100
Subject: [PATCH 7/9] Improve the tests by using a helper function for
modifying the input file and removed the redundant code
---
pyproject.toml | 2 +-
test/conftest.py | 4 +-
test/test_file_validator/test_forms.py | 89 +++++++++++++-------------
3 files changed, 47 insertions(+), 48 deletions(-)
diff --git a/pyproject.toml b/pyproject.toml
index 981838d..4650bd8 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -38,7 +38,7 @@ line-length = 120
ban-relative-imports = "all"
[tool.ruff.lint.per-file-ignores]
-"test/*" = ["S101", "ARG001",]
+"test/*" = ["ARG001", "D", "S101", "PLR2004", "PLR0917", "PLR0914", "PLC0415", "PLC2701"]
[tool.pytest.ini_options]
DJANGO_SETTINGS_MODULE = "test.settings"
diff --git a/test/conftest.py b/test/conftest.py
index 74bbc11..f91236c 100644
--- a/test/conftest.py
+++ b/test/conftest.py
@@ -9,7 +9,7 @@ from sage_validation.file_validator.models import MeoCostCentres, MeoValidSuppli
@pytest.fixture
-def sample_input_file():
+def sample_input_file() -> SimpleUploadedFile:
"""Create a sample valid CSV file for testing."""
csv_headers_list = [
"AccountNumber",
@@ -96,7 +96,7 @@ def sample_input_file():
return SimpleUploadedFile("test.csv", f"{csv_headers}\n{csv_content}".encode(), content_type="text/csv")
@pytest.fixture
-def mock_meo_database(mocker):
+def mock_meo_database(mocker: MagicMock)-> None:
"""Mock the meo database since it's read-only."""
fake = Faker()
diff --git a/test/test_file_validator/test_forms.py b/test/test_file_validator/test_forms.py
index b169538..b5df108 100644
--- a/test/test_file_validator/test_forms.py
+++ b/test/test_file_validator/test_forms.py
@@ -1,19 +1,46 @@
"""Tests for the file_validator forms."""
import csv
import io
+from unittest.mock import MagicMock
from django.core.files.uploadedfile import SimpleUploadedFile
from sage_validation.file_validator.forms import CSVUploadForm
-def test_valid_csv_upload(sample_input_file, mock_meo_database):
+def create_modified_csv(sample_file: SimpleUploadedFile, modifications: dict[str, str]) -> SimpleUploadedFile:
+ """
+ Modify specific fields in the first row of a CSV file and return a new SimpleUploadedFile.
+
+ Args:
+ sample_file (SimpleUploadedFile): The original CSV file.
+ modifications (dict): Dictionary of column names to modified values.
+
+ Returns:
+ SimpleUploadedFile: The modified CSV file.
+ """
+ csv_content = sample_file.read().decode("utf-8").splitlines()
+ reader = csv.DictReader(csv_content)
+ rows = list(reader)
+
+ for key, value in modifications.items():
+ rows[0][key] = value
+
+ output = io.StringIO()
+ writer = csv.DictWriter(output, fieldnames=reader.fieldnames or [])
+ writer.writeheader()
+ writer.writerows(rows)
+
+ return SimpleUploadedFile("test_modified.csv", output.getvalue().encode("utf-8"), content_type="text/csv")
+
+
+def test_valid_csv_upload(sample_input_file: SimpleUploadedFile, mock_meo_database: MagicMock) -> None:
"""Test CSV upload with valid data."""
form = CSVUploadForm(files={"file": sample_input_file})
assert form.is_valid(), f"Form errors: {form.errors}"
-def test_invalid_file_extension():
+def test_invalid_file_extension() -> None:
"""Test form rejects non-CSV files."""
bad_file = SimpleUploadedFile("test.txt", b"Some text content", content_type="text/plain")
form = CSVUploadForm(files={"file": bad_file})
@@ -21,7 +48,7 @@ def test_invalid_file_extension():
assert "File must be in CSV format." in form.errors["file"]
-def test_missing_required_columns():
+def test_missing_required_columns() -> None:
"""Test form rejects CSV missing required headers."""
invalid_csv = SimpleUploadedFile("test.csv", b"AccountNumber,CBAccountNumber\n12345,54321", content_type="text/csv")
form = CSVUploadForm(files={"file": invalid_csv})
@@ -29,43 +56,19 @@ def test_missing_required_columns():
assert "Missing required columns" in str(form.errors)
-def test_source_and_trader_type_validation(sample_input_file, mock_meo_database):
+def test_source_and_trader_type_validation(sample_input_file: SimpleUploadedFile, mock_meo_database: MagicMock) -> None:
"""Test validation for Source and SYSTraderTranType columns."""
- csv_content = sample_input_file.read().decode("utf-8").splitlines()
- reader = csv.DictReader(csv_content)
- rows = list(reader)
-
- # Modify the first row's "Source" and "SYSTraderTranType" values
- rows[0]["Source"] = "90" # Change Source from 80 to 90
- rows[0]["SYSTraderTranType"] = "5" # Change SYSTraderTranType from 4 to 5
-
- # Rebuild the CSV with modified values
- output = io.StringIO()
- writer = csv.DictWriter(output, fieldnames=reader.fieldnames)
- writer.writeheader()
- writer.writerows(rows)
- modified_file = SimpleUploadedFile("test_modified.csv", output.getvalue().encode("utf-8"), content_type="text/csv")
-
+ modified_file = create_modified_csv(sample_input_file, {"Source": "90", "SYSTraderTranType": "5"})
form = CSVUploadForm(files={"file": modified_file})
assert not form.is_valid()
assert "Row 1: 'Source' must be 80" in form.errors["file"][0]
assert "Row 1: 'SYSTraderTranType' must be 4" in form.errors["file"][1]
-def test_validate_nominal_analysis_account(sample_input_file, mock_meo_database):
+def test_validate_nominal_analysis_account(sample_input_file: SimpleUploadedFile, mock_meo_database: MagicMock) -> None:
"""Test validation for nominal analysis account."""
- csv_content = sample_input_file.read().decode("utf-8").splitlines()
- reader = csv.DictReader(csv_content)
- rows = list(reader)
- rows[0]["NominalAnalysisNominalAnalysisNarrative/1"] = "Invalid Name"
-
- # Rebuild the CSV with modified values
- output = io.StringIO()
- writer = csv.DictWriter(output, fieldnames=reader.fieldnames)
- writer.writeheader()
- writer.writerows(rows)
- modified_file = SimpleUploadedFile("test_modified.csv", output.getvalue().encode("utf-8"), content_type="text/csv")
-
+ modified_file = create_modified_csv(sample_input_file,
+ {"NominalAnalysisNominalAnalysisNarrative/1": "Invalid Name"})
form = CSVUploadForm(files={"file": modified_file})
assert not form.is_valid()
assert form.errors["file"][0] == (
@@ -73,20 +76,16 @@ def test_validate_nominal_analysis_account(sample_input_file, mock_meo_database)
", but found 'Invalid Name'.")
-def test_validate_nc_cc_dep_combination_against_meo_sage_account(sample_input_file, mock_meo_database):
+def test_validate_nc_cc_dep_combination_against_meo_sage_account(
+ sample_input_file: SimpleUploadedFile, mock_meo_database: MagicMock
+) -> None:
"""Test validation for nominal analysis fields against MEO valid Sage accounts."""
- csv_content = sample_input_file.read().decode("utf-8").splitlines()
- reader = csv.DictReader(csv_content)
- rows = list(reader)
-
- rows[0]["NominalAnalysisNominalCostCentre/1"] = "Invalid_CC"
- rows[0]["NominalAnalysisNominalAccountNumber/1"] = "Invalid_Account"
-
- output = io.StringIO()
- writer = csv.DictWriter(output, fieldnames=reader.fieldnames)
- writer.writeheader()
- writer.writerows(rows)
- modified_file = SimpleUploadedFile("test_modified.csv", output.getvalue().encode("utf-8"), content_type="text/csv")
+ modified_file = create_modified_csv(
+ sample_input_file,
+ {
+ "NominalAnalysisNominalCostCentre/1": "Invalid_CC",
+ "NominalAnalysisNominalAccountNumber/1": "Invalid_Account"
+ })
form = CSVUploadForm(files={"file": modified_file})
assert not form.is_valid()
--
GitLab
From 917f8856290864bd0c8ee57e9f9fff61b7f51a4f Mon Sep 17 00:00:00 2001
From: Neda Moeini <neda.moeini@geant.org>
Date: Fri, 28 Feb 2025 11:39:57 +0100
Subject: [PATCH 8/9] Replace FormView with APIView
---
requirements.txt | 1 +
.../file_validator/templates/upload.html | 58 +++++------------
sage_validation/file_validator/urls.py | 7 ++-
sage_validation/file_validator/views.py | 62 +++++++++----------
sage_validation/templates/index.html | 2 +-
5 files changed, 50 insertions(+), 80 deletions(-)
diff --git a/requirements.txt b/requirements.txt
index 9732a52..4edebd6 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,5 @@
Django==5.0.11
+django-rest-framework
ruff
mypy
tox
diff --git a/sage_validation/file_validator/templates/upload.html b/sage_validation/file_validator/templates/upload.html
index 4a595da..ec8619d 100644
--- a/sage_validation/file_validator/templates/upload.html
+++ b/sage_validation/file_validator/templates/upload.html
@@ -40,21 +40,18 @@
</div>
</div>
-
<script>
- const form = document.getElementById('uploadForm');
- const fileInput = document.getElementById('fileInput');
- const errorSection = document.getElementById('errorSection');
- const errorList = document.getElementById('errorList');
- const successSection = document.getElementById('successSection');
- const successMessage = document.getElementById('successMessage');
- const downloadSection = document.getElementById('downloadSection');
- const downloadLink = document.getElementById('downloadLink');
-
- form.addEventListener('submit', async function (e) {
+ document.getElementById('uploadForm').addEventListener('submit', async function (e) {
e.preventDefault();
- // Clear previous messages
+ const fileInput = document.getElementById('fileInput');
+ const errorSection = document.getElementById('errorSection');
+ const errorList = document.getElementById('errorList');
+ const successSection = document.getElementById('successSection');
+ const successMessage = document.getElementById('successMessage');
+ const downloadSection = document.getElementById('downloadSection');
+ const downloadLink = document.getElementById('downloadLink');
+
errorList.innerHTML = '';
successMessage.innerHTML = '';
errorSection.classList.add('hidden');
@@ -65,7 +62,7 @@
formData.append('file', fileInput.files[0]);
try {
- const response = await fetch('', {
+ const response = await fetch("{% url 'upload-file' %}", {
method: 'POST',
body: formData,
headers: {
@@ -81,38 +78,13 @@
downloadLink.href = result.download_url;
downloadSection.classList.remove('hidden');
} else if (response.status === 400 && result.status === 'error') {
- errorList.innerHTML = '';
-
- if (Array.isArray(result.errors)) {
- result.errors.forEach(errorObj => {
- if (typeof errorObj === 'string') {
- const li = document.createElement('li');
- li.textContent = errorObj;
- errorList.appendChild(li);
- } else {
- for (const [field, messages] of Object.entries(errorObj)) {
- messages.forEach(message => {
- const li = document.createElement('li');
- li.textContent = `${field}: ${message}`;
- errorList.appendChild(li);
- });
- }
- }
+ for (const [field, messages] of Object.entries(result.errors)) {
+ messages.forEach(message => {
+ const li = document.createElement('li');
+ li.textContent = `${field}: ${message}`;
+ errorList.appendChild(li);
});
- } else if (typeof result.errors === 'object') {
- for (const [field, messages] of Object.entries(result.errors)) {
- messages.forEach(message => {
- const li = document.createElement('li');
- li.textContent = `${field}: ${message}`;
- errorList.appendChild(li);
- });
- }
- } else {
- const li = document.createElement('li');
- li.textContent = result.errors;
- errorList.appendChild(li);
}
-
errorSection.classList.remove('hidden');
}
} catch (error) {
diff --git a/sage_validation/file_validator/urls.py b/sage_validation/file_validator/urls.py
index a5c0d9f..ee5ae78 100644
--- a/sage_validation/file_validator/urls.py
+++ b/sage_validation/file_validator/urls.py
@@ -2,9 +2,10 @@
from django.urls import path
-from sage_validation.file_validator.views import CSVExportView, CSVUploadView
+from sage_validation.file_validator.views import CSVExportAPIView, CSVUploadAPIView, upload_page_view
urlpatterns = [
- path("upload/", CSVUploadView.as_view(), name="upload-file"),
- path("export/", CSVExportView.as_view(), name="export-file"),
+ path("upload-page/", upload_page_view, name="upload-page"),
+ path("api/upload/", CSVUploadAPIView.as_view(), name="upload-file"),
+ path("api/export/", CSVExportAPIView.as_view(), name="export-file"),
]
diff --git a/sage_validation/file_validator/views.py b/sage_validation/file_validator/views.py
index 99efd17..bfa80c2 100644
--- a/sage_validation/file_validator/views.py
+++ b/sage_validation/file_validator/views.py
@@ -1,14 +1,15 @@
"""Views for the file_validator app."""
import csv
import io
-from typing import Any
-from django.http import HttpRequest, HttpResponse, JsonResponse
+from django.http import HttpRequest, HttpResponse
from django.shortcuts import render
from django.urls import reverse_lazy
from django.utils import timezone
-from django.views.generic.base import View
-from django.views.generic.edit import FormView
+from rest_framework import status
+from rest_framework.request import Request
+from rest_framework.response import Response
+from rest_framework.views import APIView
from sage_validation.file_validator.forms import CSVUploadForm
from sage_validation.file_validator.models import MeoCostCentres, XxData
@@ -19,46 +20,41 @@ def index_view(request: HttpRequest) -> HttpResponse:
return render(request, "index.html")
-class CSVUploadView(FormView):
- """View for uploading a CSV file."""
+def upload_page_view(request: HttpRequest) -> HttpResponse:
+ """Render the file upload page."""
+ return render(request, "upload.html")
- template_name = "upload.html"
- form_class = CSVUploadForm
- success_url = reverse_lazy("upload-file")
- def get_context_data(self, **kwargs: dict[str, Any]) -> dict[str, Any]:
- """Render the form with no error message on GET request."""
- context = super().get_context_data(**kwargs)
- context["error"] = None
- context["message"] = None
- return context
+class CSVUploadAPIView(APIView):
+ """API view for uploading a CSV file."""
+
+ def post(self, request: Request) -> Response:
+ """Handle CSV upload and validation."""
+ form = CSVUploadForm(data=request.data, files=request.FILES)
+
+ if not form.is_valid():
+ return Response({"status": "error", "errors": form.errors}, status=status.HTTP_400_BAD_REQUEST)
- def form_valid(self, form: CSVUploadForm) -> JsonResponse:
- """Handle the CSV validation, store valid data, and prepare for export."""
csv_file = form.cleaned_data["file"]
csv_file.seek(0)
decoded_file = csv_file.read().decode("utf-8").strip()
if not decoded_file:
- return JsonResponse({"status": "error", "message": "Uploaded file is empty."}, status=400)
+ return Response({"status": "error", "message": "Uploaded file is empty."},
+ status=status.HTTP_400_BAD_REQUEST)
reader = csv.DictReader(io.StringIO(decoded_file))
csv_data: list[dict[str, str]] = list(reader)
updated_data = self.update_fields(csv_data)
+ request.session["validated_csv"] = updated_data
+ request.session.modified = True
- self.request.session["validated_csv"] = updated_data
- self.request.session.modified = True
-
- return JsonResponse({
+ return Response({
"status": "success",
"message": "File successfully uploaded and processed.",
"download_url": reverse_lazy("export-file")
- })
-
- def form_invalid(self, form: CSVUploadForm) -> JsonResponse:
- """Handle the form when it is invalid."""
- return JsonResponse({"status": "error", "errors": form.errors}, status=400)
+ }, status=status.HTTP_200_OK)
@staticmethod
def update_fields(csv_data: list[dict[str, str]]) -> list[dict[str, str]]:
@@ -91,21 +87,21 @@ class CSVUploadView(FormView):
row[f"NominalAnalysisNominalAccountNumber/{repeat}"] = (
xx_data[0] if cc_type == "Project" else xx_data[1]
)
-
repeat += 1
return csv_data
-class CSVExportView(View):
- """View for exporting the updated CSV file."""
+class CSVExportAPIView(APIView):
+ """API view for exporting the updated CSV file."""
- def get(self, request: HttpRequest) -> HttpResponse:
- """Generate a downloadable CSV file with updated values."""
+ def get(self, request: Request) -> Response:
+ """Return processed CSV as a downloadable response."""
csv_data: list[dict[str, str]] = request.session.get("validated_csv", [])
if not csv_data:
- return HttpResponse("No data available for export.", status=400)
+ return Response({"status": "error", "message": "No data available for export."},
+ status=status.HTTP_400_BAD_REQUEST)
response = HttpResponse(content_type="text/csv")
response["Content-Disposition"] = 'attachment; filename="updated_file.csv"'
diff --git a/sage_validation/templates/index.html b/sage_validation/templates/index.html
index 7d7376e..72db6b8 100644
--- a/sage_validation/templates/index.html
+++ b/sage_validation/templates/index.html
@@ -8,7 +8,7 @@
<h1 class="text-5xl md:text-6xl font-bold mb-12 text-blue-900">Welcome to Sage Validation</h1>
<p class="text-xl md:text-2xl mb-16 text-gray-700">Click the button below to upload your file for validation.</p>
- <a href="{% url "upload-file" %}"
+ <a href="{% url "upload-page" %}"
class="inline-flex py-4 px-16 bg-blue-600 text-white text-lg md:text-xl font-bold rounded-full shadow-lg transition-transform transform hover:scale-105 focus:outline-none focus:ring-4 focus:ring-blue-300 focus:ring-opacity-50">
Upload File
</a>
--
GitLab
From cdbcc75afbafd4258a7c56ec63559a287c6106bd Mon Sep 17 00:00:00 2001
From: Neda Moeini <neda.moeini@geant.org>
Date: Fri, 28 Feb 2025 13:07:28 +0100
Subject: [PATCH 9/9] Add more tests for endpoints
---
sage_validation/settings.py | 2 +-
test/conftest.py | 6 +++
.../test_file_validator_endpoints.py | 52 +++++++++++++++++++
3 files changed, 59 insertions(+), 1 deletion(-)
create mode 100644 test/test_file_validator/test_file_validator_endpoints.py
diff --git a/sage_validation/settings.py b/sage_validation/settings.py
index 1a00d60..353594e 100644
--- a/sage_validation/settings.py
+++ b/sage_validation/settings.py
@@ -18,7 +18,7 @@ BASE_DIR = Path(__file__).resolve().parent
# See https://docs.djangoproject.com/en/5.1/howto/deployment/checklist/
# SECURITY WARNING: keep the secret key used in production secret!
-SECRET_KEY = os.getenv("SECRET_KEY")
+SECRET_KEY = os.getenv("SECRET_KEY", "test-secret-key")
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True
diff --git a/test/conftest.py b/test/conftest.py
index f91236c..f32a594 100644
--- a/test/conftest.py
+++ b/test/conftest.py
@@ -4,6 +4,7 @@ from unittest.mock import MagicMock
import pytest
from django.core.files.uploadedfile import SimpleUploadedFile
from faker import Faker
+from rest_framework.test import APIClient
from sage_validation.file_validator.models import MeoCostCentres, MeoValidSuppliers, XxData
@@ -139,3 +140,8 @@ def mock_meo_database(mocker: MagicMock)-> None:
nominal_mock.filter.return_value.exists.return_value = True
mocker.patch("sage_validation.file_validator.models.MeoNominal.objects.using", return_value=nominal_mock)
+
+@pytest.fixture
+def api_client() -> APIClient:
+ """Fixture to return Django API test client."""
+ return APIClient()
diff --git a/test/test_file_validator/test_file_validator_endpoints.py b/test/test_file_validator/test_file_validator_endpoints.py
new file mode 100644
index 0000000..3193a62
--- /dev/null
+++ b/test/test_file_validator/test_file_validator_endpoints.py
@@ -0,0 +1,52 @@
+from unittest.mock import MagicMock
+
+import pytest
+from django.core.files.uploadedfile import SimpleUploadedFile
+from django.urls.base import reverse
+from rest_framework.test import APIClient
+
+UPLOAD_FILE_URL = reverse("upload-file")
+
+
+@pytest.mark.django_db
+def test_csv_upload_valid(
+ api_client: APIClient, sample_input_file: SimpleUploadedFile, mock_meo_database: MagicMock
+) -> None:
+ """Test that a valid CSV upload succeeds."""
+ response = api_client.post(UPLOAD_FILE_URL, {"file": sample_input_file}, format="multipart")
+
+ assert response.status_code == 200
+ assert response.json()["status"] == "success"
+ assert "download_url" in response.json()
+
+
+@pytest.mark.django_db
+def test_csv_upload_invalid_extension(api_client: APIClient) -> None:
+ """Test that uploading a non-CSV file fails."""
+ bad_file = SimpleUploadedFile("test.txt", b"Invalid content", content_type="text/plain")
+
+ response = api_client.post(UPLOAD_FILE_URL, {"file": bad_file}, format="multipart")
+
+ assert response.status_code == 400
+ assert response.json()["status"] == "error"
+ assert "errors" in response.json()
+
+
+@pytest.mark.django_db
+def test_csv_export_with_data(api_client: APIClient) -> None:
+ """Test exporting a processed CSV file."""
+ url = reverse("export-file")
+
+ # Simulate session data
+ session = api_client.session
+ session["validated_csv"] = [
+ {"AccountNumber": "12345", "TransactionDate": "01/03/2024", "NominalAnalysisNominalAccountNumber/1": "N100"}
+ ]
+ session.save()
+
+ response = api_client.get(url)
+
+ assert response.status_code == 200
+ assert response["Content-Disposition"] == 'attachment; filename="updated_file.csv"'
+ assert b"AccountNumber,TransactionDate,NominalAnalysisNominalAccountNumber/1" in response.content
+ assert b"12345,01/03/2024,N100" in response.content
--
GitLab