From e63af3e66dcfa85719d871bdbf2457a3242f0635 Mon Sep 17 00:00:00 2001
From: GotthardG <51994228+GotthardG@users.noreply.github.com>
Date: Wed, 15 Jan 2025 14:23:51 +0100
Subject: [PATCH] ``` Add duplicate detection for spreadsheet data processing
Implemented logic to detect and handle duplicate 'positioninpuck' entries within the same puck during spreadsheet processing. Updated backend to validate duplicates and provide detailed error messages. Enhanced frontend to visually highlight duplicate errors and allow better user feedback during cell editing.
```
---
backend/app/services/spreadsheet_service.py | 44 +++++++++++---
frontend/src/components/SpreadsheetTable.tsx | 60 +++++++++++++-------
2 files changed, 77 insertions(+), 27 deletions(-)
diff --git a/backend/app/services/spreadsheet_service.py b/backend/app/services/spreadsheet_service.py
index c3a43fa..f0a58a7 100644
--- a/backend/app/services/spreadsheet_service.py
+++ b/backend/app/services/spreadsheet_service.py
@@ -152,9 +152,7 @@ class SampleSpreadsheetImporter:
model = []
errors = []
raw_data = []
- headers = []
- # Skip the first 3 rows
rows = list(sheet.iter_rows(min_row=4, values_only=True))
logger.debug(f"Starting to process {len(rows)} rows from the sheet")
@@ -200,6 +198,8 @@ class SampleSpreadsheetImporter:
"chiphiangles",
]
+ duplicate_check = {}
+
for index, row in enumerate(rows):
if not any(row):
logger.debug(f"Skipping empty row at index {index}")
@@ -241,13 +241,12 @@ class SampleSpreadsheetImporter:
corrected = True
defaulted_columns.append(column_name)
- # Update the record with cleaned value (store only the cleaned part,
- # not the tuple)
+ # Update the record with cleaned value
record[column_name] = cleaned_value
except (ValueError, TypeError) as e:
logger.error(
- f"Validation error for row {index + 4}"
- f", column '{column_name}': {str(e)}"
+ f"Validation error for row {index + 4},"
+ f" column '{column_name}': {str(e)}"
)
errors.append(
{
@@ -258,6 +257,37 @@ class SampleSpreadsheetImporter:
}
)
+ # Validate duplicate 'positioninpuck' within the same puck
+ dewarname = record.get("dewarname")
+ puckname = record.get("puckname")
+ positioninpuck = record.get("positioninpuck")
+
+ if (
+ dewarname and puckname and positioninpuck is not None
+ ): # Only check if all required fields exist
+ duplicate_key = f"{dewarname}-{puckname}"
+ if duplicate_key not in duplicate_check:
+ duplicate_check[duplicate_key] = set()
+
+ if positioninpuck in duplicate_check[duplicate_key]:
+ # Add error for duplicate position in the same puck
+ logger.warning(
+ f"Duplicate position '{positioninpuck}' found in puck"
+ f" '{puckname}' (dewar: '{dewarname}')"
+ )
+ errors.append(
+ {
+ "row": index + 4, # Adjust row number for 1-based indexing
+ "column": "positioninpuck", # The problematic column
+ "value": positioninpuck, # The value causing the issue
+ "message": f"Duplicate position '{positioninpuck}'"
+ f" found in puck '{puckname}' of dewar '{dewarname}'.",
+ }
+ )
+
+ else:
+ duplicate_check[duplicate_key].add(positioninpuck)
+
# Build metadata for the row
raw_data.append(
{
@@ -267,9 +297,7 @@ class SampleSpreadsheetImporter:
defaulted_columns
), # True if any defaults were applied
"corrected": corrected, # True if any value was corrected
- # List of corrected columns (if any)
"corrected_columns": corrected_columns,
- # List of defaulted columns (if any)
"defaulted_columns": defaulted_columns,
}
)
diff --git a/frontend/src/components/SpreadsheetTable.tsx b/frontend/src/components/SpreadsheetTable.tsx
index 366395a..a7fa0bc 100644
--- a/frontend/src/components/SpreadsheetTable.tsx
+++ b/frontend/src/components/SpreadsheetTable.tsx
@@ -83,15 +83,18 @@ const SpreadsheetTable = ({
}, [selectedShipment]);
const generateErrorMap = (errorsList) => {
- const errorMap = new Map();
- if (Array.isArray(errorsList)) {
- errorsList.forEach((error) => {
- const key = `${error.row}-${headers[error.cell]}`;
- errorMap.set(key, error.message);
- });
- }
- return errorMap;
- };
+ const errorMap = new Map();
+ if (Array.isArray(errorsList)) {
+ errorsList.forEach((error) => {
+ const colIndex = headers.findIndex(header => header === error.column);
+ if (colIndex > -1) {
+ const key = `${error.row}-${headers[colIndex]}`;
+ errorMap.set(key, error.message);
+ }
+ });
+ }
+ return errorMap;
+ };
const errorMap = generateErrorMap(localErrors);
@@ -139,7 +142,6 @@ const SpreadsheetTable = ({
if (response && response.is_valid !== undefined) {
if (response.is_valid) {
- // If valid, update the value (and use corrected_value if returned)
const correctedValue = response.corrected_value ?? newValue;
currentRow.data[colIndex] = correctedValue;
updatedRawData[rowIndex] = currentRow;
@@ -148,7 +150,12 @@ const SpreadsheetTable = ({
// Remove the error and mark as non-editable
const updatedErrors = localErrors.filter(
- (error) => !(error.row === currentRow.row_num && error.cell === colIndex)
+ (error) =>
+ !(
+ error.row === currentRow.row_num &&
+ error.cell === colIndex &&
+ error.message.toLowerCase().includes("duplicate position")
+ )
);
setLocalErrors(updatedErrors); // Update error list
@@ -188,7 +195,11 @@ const SpreadsheetTable = ({
handleCellEdit(rowIndex, colIndex);
};
- const allCellsValid = () => nonEditableCells.size === raw_data.length * headers.length;
+ const allCellsValid = () =>
+ nonEditableCells.size === raw_data.length * headers.length &&
+ !localErrors.some((error) =>
+ error.message.toLowerCase().includes("duplicate position")
+ );
const fieldToCol = {
'dewarname': 0,
@@ -514,6 +525,7 @@ const SpreadsheetTable = ({
const key = `${row.row_num}-${header}`;
const errorMessage = errorMap.get(key);
const isInvalid = !!errorMessage;
+ const isDuplicateError = errorMessage?.toLowerCase().includes("duplicate position"); // Detect duplicate-specific messages
const cellValue = row.data[colIndex];
const editingValue = editingCell[`${rowIndex}-${colIndex}`];
const isCellCorrected = row.corrected_columns?.includes(header); // Use corrected metadata
@@ -525,13 +537,21 @@ const SpreadsheetTable = ({
align="center"
style={{
backgroundColor: isDefaultAssigned
- ? "#e6fbe6" // Light green
+ ? "#e6fbe6" // Light green for default values
: isCellCorrected
- ? "#fff8e1" // Light yellow
- : "transparent",
- color: isDefaultAssigned ? "#1b5e20" : "inherit",
- fontWeight: isCellCorrected || isDefaultAssigned ? "bold" : "normal",
- cursor: isInvalid ? "pointer" : "default",
+ ? "#fff8e1" // Light yellow for corrections
+ : isDuplicateError
+ ? "#fdecea" // Light red for duplicate errors
+ : "transparent", // No specific color for valid cells
+ color: isDefaultAssigned
+ ? "#1b5e20" // Dark green for default values
+ : isDuplicateError
+ ? "#d32f2f" // Bright red for duplicates
+ : "inherit",
+ fontWeight: isDefaultAssigned || isCellCorrected || isDuplicateError
+ ? "bold"
+ : "normal",
+ cursor: isInvalid ? "pointer" : "default", // Allow editing invalid cells
}}
>
<Tooltip
@@ -540,7 +560,9 @@ const SpreadsheetTable = ({
? "This value was automatically assigned as a default."
: isCellCorrected
? `Field "${header}" was auto-corrected.`
- : errorMessage || ""
+ : isDuplicateError
+ ? `Duplicate value detected in "${header}". Please ensure values are unique in this column.`
+ : errorMessage || ""
}
arrow
disableHoverListener={!isDefaultAssigned && !isCellCorrected && !isInvalid}
--
GitLab