Skip to content
Snippets Groups Projects
Commit 28d3faca authored by geant-release-service's avatar geant-release-service
Browse files

Finished release 0.4.

parents 7f1a6fe2 d5714d60
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 211 additions and 94 deletions
Changelog.md
+ 3
− 0
View file @ 28d3faca
# Changelog
## [0.3] - 2025-02-13
- Added CSV Export and Modify Functionality
## [0.2] - 2025-01-27
- Added new validations
......
sage_validation/file_validator/forms.py
+ 55
− 36
View file @ 28d3faca
"""Forms for the file_validator app."""
import csv
from collections.abc import Iterable, Sequence
from collections.abc import Sequence
from typing import ClassVar
from django import forms
from django.core.files.uploadedfile import UploadedFile
from sage_validation.file_validator.models import Meocostcentres, Meovalidsageaccounts, PlAccountCodes, XxData
from sage_validation.file_validator.models import (
MeoCostCentres,
MeoNominal,
MeoValidSageAccounts,
MeoValidSuppliers,
XxData,
)
class CSVUploadForm(forms.Form):
......@@ -72,7 +78,6 @@ class CSVUploadForm(forms.Form):
self._validate_headers(fieldnames)
error_list = []
# Step 3: Validate 'Source' and 'SYSTraderTranType' values
data = list(reader)
error_list.extend(self._validate_source_and_trader_type(data))
error_list.extend(self._validate_nominal_analysis_account(data))
......@@ -121,7 +126,7 @@ class CSVUploadForm(forms.Form):
raise forms.ValidationError(msg)
@staticmethod
def _validate_source_and_trader_type(data: Iterable[dict]) -> list:
def _validate_source_and_trader_type(data: list[dict]) -> list:
"""Validate that 'Source' is always 80 and 'SYSTraderTranType' is always 4."""
errors = []
......@@ -135,14 +140,14 @@ class CSVUploadForm(forms.Form):
return errors
@staticmethod
def _validate_nominal_analysis_account(data: Iterable[dict]) -> list[str]:
def _validate_nominal_analysis_account(data: list[dict]) -> list[str]:
"""Validate that 'AccountNumber' matches the name in 'NominalAnalysisNominalAnalysisNarrative/1'.
This only checks the first group of NominalAnalysis columns. A list of codes/names
is fetched from the database for validation (from the 'PL Account Codes' table).
Args:
data (Iterable[dict]): The rows of data to validate.
data (list[dict]): The rows of data to validate.
Returns:
List[str]: A list of error messages, if any.
......@@ -151,8 +156,8 @@ class CSVUploadForm(forms.Form):
errors = []
account_code_map = {
obj.pl_account_code: obj.pl_account_name
for obj in PlAccountCodes.objects.using("meo").all() # type: ignore[attr-defined]
obj.supplier_account_number: obj.supplier_account_name
for obj in MeoValidSuppliers.objects.using("meo").all() # type: ignore[attr-defined]
}
for index, row in enumerate(data, start=1):
......@@ -174,12 +179,15 @@ class CSVUploadForm(forms.Form):
return errors
@staticmethod
def _validate_nc_cc_dep_combination_against_meo_sage_account(data: Iterable[dict]) -> list[str]:
"""Validate that the combination of 'AccountCostCentre', 'AccountDepartment', and 'AccountNumber' exists in MEO.
def _validate_nc_cc_dep_combination_against_meo_sage_account(self, data: list[dict]) -> list[str]:
"""Validate that all nominal analysis fields exist in MEO.
This includes 'NominalAnalysisNominalCostCentre/{N}', 'NominalAnalysisNominalDepartment/{N}',
and 'NominalAnalysisNominalAccountNumber/{N}'.
Args:
data (Iterable[dict]): The rows of data to validate.
data (list[dict]): The rows of data to validate.
Returns:
List[str]: A list of error messages, if any.
......@@ -188,41 +196,52 @@ class CSVUploadForm(forms.Form):
errors = []
cost_centre_map = {
obj.cc: obj.cctype for obj in Meocostcentres.objects.using("meo").all()
obj.cc: obj.cc_type for obj in MeoCostCentres.objects.using("meo").all()
}
xx_data_map = {
obj.xx_value: (obj.project, obj.overhead) for obj in XxData.objects.using("meo").all()
}
fieldnames = list(data[0].keys())
max_repeat = self._get_max_repeat(fieldnames, "NominalAnalysisNominalCostCentre")
for index, row in enumerate(data, start=1):
cc = row.get("NominalAnalysisNominalCostCentre/1")
dep = row.get("NominalAnalysisNominalDepartment/1")
nominal_account_name = row.get("NominalAnalysisNominalAccountNumber/1")
for repeat in range(1, max_repeat + 1):
cc_field = f"NominalAnalysisNominalCostCentre/{repeat}"
dep_field = f"NominalAnalysisNominalDepartment/{repeat}"
nominal_account_field = f"NominalAnalysisNominalAccountNumber/{repeat}"
if not cc or not dep or not nominal_account_name:
continue
cc = row.get(cc_field)
dep = row.get(dep_field)
nominal_account_name = row.get(nominal_account_field)
cc_type = cost_centre_map.get(cc)
if not cc_type:
errors.append(f"Row {index}: 'NominalAnalysisNominalCostCentre/1' ({cc}) is not a valid cost centre.")
continue
if not cc or not dep or not nominal_account_name:
continue
xx_data = xx_data_map.get(nominal_account_name)
if not xx_data:
errors.append(
f"Row {index}: 'NominalAnalysisNominalAccountNumber/1' ({nominal_account_name}) is not valid.")
continue
cc_type = cost_centre_map.get(cc)
if not cc_type:
errors.append(f"Row {index}: '{cc_field}' ({cc}) is not a valid cost centre.")
continue
nc = xx_data[0] if cc_type == "Project" else xx_data[1]
xx_data = xx_data_map.get(nominal_account_name)
if not Meovalidsageaccounts.objects.using("meo").filter(
accountcostcentre=cc, accountdepartment=dep, accountnumber=nc
).exists():
errors.append(
f"Row {index}: The combination of 'NominalAnalysisNominalCostCentre/1' ({cc}), "
f"'NominalAnalysisNominalDepartment/1' ({dep}), and 'NominalAnalysisNominalAccountNumber/1' "
f"({nominal_account_name}) does not exist in MEO valid Sage accounts."
)
if xx_data:
nc = xx_data[0] if cc_type == "Project" else xx_data[1]
elif MeoNominal.objects.using("meo").filter(nom=nominal_account_name).exists():
nc = nominal_account_name
else:
errors.append(f"Row {index}: '{nominal_account_field}' ({nominal_account_name}) is not valid.")
continue
if not MeoValidSageAccounts.objects.using("meo").filter(
account_cost_centre=cc, account_department=dep, account_number=nc
).exists():
errors.append(
f"Row {index}: The combination of '{cc_field}' ({cc}), "
f"'{dep_field}' ({dep}), and '{nominal_account_field}' "
f"({nominal_account_name}) does not exist in MEO valid Sage accounts."
)
return errors
sage_validation/file_validator/models.py
+ 20
− 37
View file @ 28d3faca
......@@ -2,24 +2,7 @@
from django.db import models
class PlAccountCodes(models.Model):
"""Represents profit and loss (PL) account codes.
This model maps the PL account names to their respective codes.
"""
objects = None
pl_account_name = models.TextField(db_column="PL_Account_Name")
pl_account_code = models.CharField(db_column="PL_Account_Code", primary_key=True, max_length=50)
class Meta:
"""Metaclass for the PlAccountCodes model."""
managed = False
db_table = "PL_Account_Codes"
class Meocostcentres(models.Model):
class MeoCostCentres(models.Model):
"""Represents cost centres.
This model contains data related to cost centres,
......@@ -28,68 +11,68 @@ class Meocostcentres(models.Model):
cc = models.CharField(db_column="CC", max_length=3)
cc_name = models.CharField(db_column="CC_Name", max_length=50, blank=True, null=True)
cctype = models.CharField(db_column="CCType", max_length=8)
cc_type = models.CharField(db_column="CCType", max_length=8)
id = models.IntegerField(db_column="ID", primary_key=True)
class Meta:
"""Metaclass for the Meocostcentres model."""
"""Metaclass for the MeoCostCentres model."""
managed = False
db_table = "meoCostCentres"
class Meonominal(models.Model):
class MeoNominal(models.Model):
"""View for MEO nominal codes."""
nom = models.CharField(db_column="Nom", max_length=5)
nomname = models.CharField(db_column="NomName", max_length=60, blank=True, null=True)
nomid = models.IntegerField(db_column="NomID")
nom_name = models.CharField(db_column="NomName", max_length=60, blank=True, null=True)
nom_id = models.IntegerField(db_column="NomID")
class Meta:
"""Metaclass for the Meonominal model."""
"""Metaclass for the MeoNominal model."""
managed = False
db_table = "meoNominal"
class Meovalidsageaccounts(models.Model):
class MeoValidSageAccounts(models.Model):
"""View for MEO valid Sage accounts."""
accountname = models.CharField(db_column="AccountName", max_length=60)
accountnumber = models.CharField(db_column="AccountNumber", max_length=8, blank=True, null=True)
accountcostcentre = models.CharField(db_column="AccountCostCentre", max_length=3, blank=True, null=True)
accountdepartment = models.CharField(db_column="AccountDepartment", max_length=3, blank=True, null=True)
account_name = models.CharField(db_column="AccountName", max_length=60)
account_number = models.CharField(db_column="AccountNumber", max_length=8, blank=True, null=True)
account_cost_centre = models.CharField(db_column="AccountCostCentre", max_length=3, blank=True, null=True)
account_department = models.CharField(db_column="AccountDepartment", max_length=3, blank=True, null=True)
class Meta:
"""Metaclass for the Meovalidsageaccounts model."""
"""Metaclass for the MeoValidSageAccounts model."""
managed = False
db_table = "meoValidSageAccounts"
class Meovalidsuppliers(models.Model):
class MeoValidSuppliers(models.Model):
"""View for MEO valid suppliers."""
supplieraccountnumber = models.CharField(db_column="SupplierAccountNumber", max_length=8)
supplieraccountname = models.CharField(db_column="SupplierAccountName", max_length=60)
supplier_account_number = models.CharField(db_column="SupplierAccountNumber", max_length=8, primary_key=True)
supplier_account_name = models.CharField(db_column="SupplierAccountName", max_length=60)
class Meta:
"""Metaclass for the Meovalidsuppliers model."""
"""Metaclass for the MeoValidSuppliers model."""
managed = False
db_table = "meoValidSuppliers"
class Meovalidvat(models.Model):
class MeoValidVat(models.Model):
"""View for MEO valid VAT codes."""
tax_code = models.SmallIntegerField(db_column="Tax code")
tax_code_label = models.CharField(db_column="tax code label", max_length=60)
rate = models.DecimalField(db_column="Rate", max_digits=18, decimal_places=2)
inputnominalaccount = models.CharField(db_column="InputNominalAccount", max_length=8, blank=True, null=True)
input_nominal_account = models.CharField(db_column="InputNominalAccount", max_length=8, blank=True, null=True)
class Meta:
"""Metaclass for the Meovalidvat model."""
"""Metaclass for the MeoValidVat model."""
managed = False
db_table = "meoValidVAT"
......
sage_validation/file_validator/templates/upload.html
+ 39
− 11
View file @ 28d3faca
......@@ -26,12 +26,21 @@
</div>
<!-- Success Message -->
<div id="successSection" class="hidden mt-4 bg-green-100 border border-green-400 text-green-700 p-4 rounded-lg">
<strong class="font-bold">Success:</strong>
<span id="successMessage" class="block mt-2"></span>
<div id="successSection"
class="hidden mt-4 mb-8 bg-green-100 border border-green-400 text-green-700 p-4 rounded-lg text-center">
<strong class="font-bold text-lg">Success!</strong>
<p id="successMessage" class="block mt-2 text-md"></p>
</div>
<div id="downloadSection" class="hidden mt-16 flex justify-center">
<a id="downloadLink" href="#"
class="w-full max-w-xs text-center bg-blue-600 text-white py-4 px-12 rounded-lg shadow-lg hover:bg-blue-700
focus:outline-none focus:ring focus:ring-blue-400 transition text-lg font-semibold block">
📥 Download Updated CSV
</a>
</div>
</div>
<script>
const form = document.getElementById('uploadForm');
const fileInput = document.getElementById('fileInput');
......@@ -39,6 +48,8 @@
const errorList = document.getElementById('errorList');
const successSection = document.getElementById('successSection');
const successMessage = document.getElementById('successMessage');
const downloadSection = document.getElementById('downloadSection');
const downloadLink = document.getElementById('downloadLink');
form.addEventListener('submit', async function (e) {
e.preventDefault();
......@@ -48,6 +59,7 @@
successMessage.innerHTML = '';
errorSection.classList.add('hidden');
successSection.classList.add('hidden');
downloadSection.classList.add('hidden');
const formData = new FormData();
formData.append('file', fileInput.files[0]);
......@@ -66,18 +78,35 @@
if (response.ok && result.status === 'success') {
successMessage.innerText = result.message;
successSection.classList.remove('hidden');
downloadLink.href = result.download_url;
downloadSection.classList.remove('hidden');
} else if (response.status === 400 && result.status === 'error') {
// Handle form errors from the backend
errorList.innerHTML = '';
if (Array.isArray(result.errors)) {
result.errors.forEach(errorObj => {
for (const [field, messages] of Object.entries(errorObj)) {
messages.forEach(message => {
const li = document.createElement('li');
li.textContent = `${field}: ${message}`;
errorList.appendChild(li);
});
if (typeof errorObj === 'string') {
const li = document.createElement('li');
li.textContent = errorObj;
errorList.appendChild(li);
} else {
for (const [field, messages] of Object.entries(errorObj)) {
messages.forEach(message => {
const li = document.createElement('li');
li.textContent = `${field}: ${message}`;
errorList.appendChild(li);
});
}
}
});
} else if (typeof result.errors === 'object') {
for (const [field, messages] of Object.entries(result.errors)) {
messages.forEach(message => {
const li = document.createElement('li');
li.textContent = `${field}: ${message}`;
errorList.appendChild(li);
});
}
} else {
const li = document.createElement('li');
li.textContent = result.errors;
......@@ -87,7 +116,6 @@
errorSection.classList.remove('hidden');
}
} catch (error) {
// Handle unexpected errors
const li = document.createElement('li');
li.textContent = 'An unexpected error occurred. Please try again.';
errorList.appendChild(li);
......
sage_validation/file_validator/urls.py
+ 2
− 1
View file @ 28d3faca
......@@ -2,8 +2,9 @@
from django.urls import path
from sage_validation.file_validator.views import CSVUploadView
from sage_validation.file_validator.views import CSVExportView, CSVUploadView
urlpatterns = [
path("upload/", CSVUploadView.as_view(), name="upload-file"),
path("export/", CSVExportView.as_view(), name="export-file"),
]
sage_validation/file_validator/views.py
+ 87
− 6
View file @ 28d3faca
"""Views for the file_validator app."""
import csv
import io
from typing import Any
from django.http import HttpRequest, HttpResponse, JsonResponse
from django.shortcuts import render
from django.urls import reverse_lazy
from django.utils import timezone
from django.views.generic.base import View
from django.views.generic.edit import FormView
from sage_validation.file_validator.forms import CSVUploadForm
from sage_validation.file_validator.models import MeoCostCentres, XxData
def index_view(request: HttpRequest) -> HttpResponse:
......@@ -20,17 +26,92 @@ class CSVUploadView(FormView):
form_class = CSVUploadForm
success_url = reverse_lazy("upload-file")
def get_context_data(self, **kwargs: dict) -> dict:
def get_context_data(self, **kwargs: dict[str, Any]) -> dict[str, Any]:
"""Render the form with no error message on GET request."""
context = super().get_context_data(**kwargs)
context["error"] = None
context["message"] = None
return context
def form_valid(self, form: CSVUploadForm) -> JsonResponse: # noqa: ARG002
"""Handle the CSV validation and passes appropriate success messages to the template."""
return JsonResponse({"status": "success", "message": "File is valid"})
def form_valid(self, form: CSVUploadForm) -> JsonResponse:
"""Handle the CSV validation, store valid data, and prepare for export."""
csv_file = form.cleaned_data["file"]
csv_file.seek(0)
decoded_file = csv_file.read().decode("utf-8").strip()
if not decoded_file:
return JsonResponse({"status": "error", "message": "Uploaded file is empty."}, status=400)
reader = csv.DictReader(io.StringIO(decoded_file))
csv_data: list[dict[str, str]] = list(reader)
updated_data = self.update_fields(csv_data)
self.request.session["validated_csv"] = updated_data
self.request.session.modified = True
return JsonResponse({
"status": "success",
"message": "File successfully uploaded and processed.",
"download_url": reverse_lazy("export-file")
})
def form_invalid(self, form: CSVUploadForm) -> JsonResponse:
"""Handle the form when it is invalid (e.g., wrong file type or validation errors)."""
return JsonResponse({"status": "error", "errors": [form.errors]}, status=400)
"""Handle the form when it is invalid."""
return JsonResponse({"status": "error", "errors": form.errors}, status=400)
@staticmethod
def update_fields(csv_data: list[dict[str, str]]) -> list[dict[str, str]]:
"""Automatically update specific fields before export."""
current_date: str = timezone.now().strftime("%d/%m/%Y")
xx_data_map: dict[str, tuple] = {
obj.xx_value: (obj.project, obj.overhead) for obj in XxData.objects.using("meo").all()
}
cost_centre_map: dict[str, str] = {
obj.cc: obj.cc_type for obj in MeoCostCentres.objects.using("meo").all()
}
for row in csv_data:
row["TransactionDate"] = current_date
repeat = 1
while f"NominalAnalysisNominalCostCentre/{repeat}" in row:
cc = row.get(f"NominalAnalysisNominalCostCentre/{repeat}", "")
nominal_account_name = row.get(f"NominalAnalysisNominalAccountNumber/{repeat}", "")
if not cc or not nominal_account_name:
repeat += 1
continue
cc_type = cost_centre_map.get(cc, "")
xx_data = xx_data_map.get(nominal_account_name)
if xx_data:
row[f"NominalAnalysisNominalAccountNumber/{repeat}"] = (
xx_data[0] if cc_type == "Project" else xx_data[1]
)
repeat += 1
return csv_data
class CSVExportView(View):
"""View for exporting the updated CSV file."""
def get(self, request: HttpRequest) -> HttpResponse:
"""Generate a downloadable CSV file with updated values."""
csv_data: list[dict[str, str]] = request.session.get("validated_csv", [])
if not csv_data:
return HttpResponse("No data available for export.", status=400)
response = HttpResponse(content_type="text/csv")
response["Content-Disposition"] = 'attachment; filename="updated_file.csv"'
writer = csv.DictWriter(response, fieldnames=csv_data[0].keys())
writer.writeheader()
writer.writerows(csv_data)
return response
sage_validation/settings.py
+ 2
− 0
View file @ 28d3faca
......@@ -89,6 +89,8 @@ DATABASES = {
'OPTIONS': {
'driver': 'ODBC Driver 18 for SQL Server',
'extra_params': 'TrustServerCertificate=Yes;'
},
},
}
......
setup.py
+ 2
− 2
View file @ 28d3faca
......@@ -3,12 +3,12 @@ from setuptools import find_packages, setup
setup(
name="sage-validation",
version="0.3",
version="0.4",
packages=find_packages(),
include_package_data=True,
install_requires=[
"Django==5.0.11",
"mssql-django",
"mssql-django==1.5",
],
extras_require={
"prod": [
......
tox.ini
+ 1
− 1
View file @ 28d3faca
[tox]
envlist = py312
envlist = py311
[testenv]
deps =
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment