From bcf3cf066755c00b8f53ca6b945119d8efbe0dad Mon Sep 17 00:00:00 2001 From: Carlos-Mesquita Date: Fri, 13 Dec 2024 21:29:54 +0000 Subject: [PATCH] ENCOA-281 --- .../UserImportSummary/ExcelError.tsx | 77 +++++ src/components/UserImportSummary/index.tsx | 298 ++++++++++++++++++ src/pages/(admin)/Lists/BatchCreateUser.tsx | 246 +++++++++++---- src/utils/excel.errors.ts | 27 ++ 4 files changed, 587 insertions(+), 61 deletions(-) create mode 100644 src/components/UserImportSummary/ExcelError.tsx create mode 100644 src/components/UserImportSummary/index.tsx create mode 100644 src/utils/excel.errors.ts diff --git a/src/components/UserImportSummary/ExcelError.tsx b/src/components/UserImportSummary/ExcelError.tsx new file mode 100644 index 00000000..20245371 --- /dev/null +++ b/src/components/UserImportSummary/ExcelError.tsx @@ -0,0 +1,77 @@ +import React from 'react'; +import { FiAlertCircle } from "react-icons/fi"; +import Dropdown from '../Dropdown'; +import { errorsByRows, ExcelError } from '@/utils/excel.errors'; + +const ParseExcelErrors: React.FC<{ errors: ExcelError[] }> = (props) => { + const errorsByRow = errorsByRows(props.errors); + + const ErrorTitle = (row: string) => ( +
+ +

+ Errors found in row {row} +

+
+ ); + + return ( +
+ {Object.entries(errorsByRow).map(([row, rowErrors]) => ( +
+ +
+ {rowErrors.required.length > 0 && ( +
+
+ Missing values: +
+
+ + On columns:  + + + {rowErrors.required.join(', ')} + +
+
+ )} + + {rowErrors.other.length > 0 && ( +
+ {rowErrors.other.map((error, index) => ( +
+
+ {error.error}: +
+
+ Value + + {error.value} + + in column + + {error.column} + + is invalid! +
+
+ ))} +
+ )} +
+ +
+ ))} +
+ ); +}; + +export default ParseExcelErrors; diff --git a/src/components/UserImportSummary/index.tsx b/src/components/UserImportSummary/index.tsx new file mode 100644 index 00000000..ad58846d --- /dev/null +++ b/src/components/UserImportSummary/index.tsx @@ -0,0 +1,298 @@ +import React, { useState, useMemo } from 'react'; +import { Card, CardHeader, CardTitle, CardContent } from '@/components/ui/card'; +import { + FaCheckCircle, + FaTimesCircle, + FaExclamationCircle, + FaInfoCircle, + FaUsers, + FaExclamationTriangle +} from 'react-icons/fa'; +import { UserImport } from '@/interfaces/IUserImport'; +import Modal from '../Modal'; +import ParseExcelErrors from './ExcelError'; +import { errorsByRows, ExcelError } from '@/utils/excel.errors'; +import UserTable from '../UserTable'; + +interface Props { + parsedExcel: { rows?: any[]; errors?: any[] }, + newUsers: UserImport[], + enlistedUsers: UserImport[], + duplicateRows?: { duplicates: DuplicatesMap, count: number } +} + +export interface DuplicatesMap { + studentID: Map; + email: Map; + passport_id: Map; + phone: Map; +} + +interface ClassroomCounts { + [key: string]: number; +} + + +const UserImportSummary: React.FC = ({ parsedExcel, newUsers, enlistedUsers, duplicateRows }) => { + const [showErrorsModal, setShowErrorsModal] = useState(false); + const [showDuplicatesModal, setShowDuplicatesModal] = useState(false); + const [showEnlistedModal, setShowEnlistedModal] = useState(false); + const [showClassroomModal, setShowClassromModal] = useState(false); + + const classroomCounts = useMemo(() => { + return newUsers.reduce((acc: ClassroomCounts, user) => { + const group = user.groupName; + acc[group] = (acc[group] || 0) + 1; + return acc; + }, {}); + }, [newUsers]); + + const errorCount = Object.entries(errorsByRows(parsedExcel.errors as ExcelError[])).length || 0; + + const fieldMapper = { + "studentID": "Student ID", + "passport_id": "Passport/National ID", + "phone": "Phone Number", + "email": "E-mail" + } + + return ( + <> + + + Import Summary + + +
+
+ + + {`${parsedExcel.rows?.length} total spreadsheet rows`} + {parsedExcel.rows && parsedExcel.rows.filter(row => row === null).length > 0 && ( + + {` (${parsedExcel.rows.filter(row => row === null).length} empty)`} + + )} + +
+
+
+ {newUsers.length > 0 ? ( + + ) : ( + + )} + {`${newUsers.length} new user${newUsers.length > 1 ? "s" : ''} to import`} +
+ {newUsers.length > 0 && ( + + )} +
+ + {enlistedUsers.length > 0 && ( +
+
+ + {`${enlistedUsers.length} already registered user${enlistedUsers.length > 1 ? "s" : ''}`} +
+ +
+ )} + + {(duplicateRows && duplicateRows.count > 0) && ( +
+
+ + {duplicateRows.count} duplicate entries in file +
+ +
+ )} + + {errorCount > 0 && ( +
+
+ + {errorCount} invalid rows +
+ +
+ )} +
+ {(enlistedUsers.length > 0 || (duplicateRows && duplicateRows.count > 0) || errorCount > 0) && ( +
+
+
+ +
+
+

+ The following will be excluded from import: +

+
    + {duplicateRows && duplicateRows.count > 0 && ( +
  • +
    + {duplicateRows.count} rows with duplicate values +
    +
    + {(Object.keys(duplicateRows.duplicates) as Array).map(field => { + const duplicates = Array.from(duplicateRows.duplicates[field].entries()) + .filter((entry) => entry[1].length > 1); + if (duplicates.length === 0) return null; + return ( +
    + {field}: rows { + duplicates.map(([_, rows]) => rows.join(', ')).join('; ') + } +
    + ); + })} +
    +
    • + )} + {errorCount > 0 && ( +
  • +
    + {errorCount} rows with invalid or missing information +
    +
    + Rows: {Object.keys(errorsByRows(parsedExcel.errors || [])).join(', ')} +
    +
    • + )} +
+
+
+
+ )} + + + + setShowErrorsModal(false)}> + <> +
+ +

Validation Errors

+
+ + + + + setShowEnlistedModal(false)}> + <> +
+ +

Already Registered Users

+
+
+ +
+ + + + setShowDuplicatesModal(false)}> + <> +
+ +

Duplicate Entries

+
+ {duplicateRows && + < div className="space-y-6"> + {(Object.keys(duplicateRows.duplicates) as Array).map(field => { + const duplicates = Array.from(duplicateRows!.duplicates[field].entries()) + .filter((entry): entry is [string, number[]] => entry[1].length > 1); + + if (duplicates.length === 0) return null; + + return ( +
+
+

+ {fieldMapper[field]} duplicates +

+ + {duplicates.length} {duplicates.length === 1 ? 'duplicate' : 'duplicates'} + +
+ +
+ {duplicates.map(([value, rows]) => ( +
+
+
+ {value} +
+ Appears in rows: + + {rows.join(', ')} + +
+
+ + {rows.length} occurrences + +
+
+ ))} +
+
+ ); + })} + + } + + + + setShowClassromModal(false)}> + <> +
+ +

Students by Classroom

+
+
+ {Object.entries(classroomCounts).map(([classroom, count]) => ( +
+
+ {classroom} +
+
+ {count} + students +
+
+ ))} +
+ + + + ); +}; + +export default UserImportSummary; diff --git a/src/pages/(admin)/Lists/BatchCreateUser.tsx b/src/pages/(admin)/Lists/BatchCreateUser.tsx index 814ae38f..7c389b8f 100644 --- a/src/pages/(admin)/Lists/BatchCreateUser.tsx +++ b/src/pages/(admin)/Lists/BatchCreateUser.tsx @@ -6,14 +6,13 @@ import { toast } from "react-toastify"; import { useFilePicker } from "use-file-picker"; import readXlsxFile from "read-excel-file"; import Modal from "@/components/Modal"; -import { BsQuestionCircleFill } from "react-icons/bs"; import { PermissionType } from "@/interfaces/permissions"; import moment from "moment"; import { checkAccess, getTypesOfUser } from "@/utils/permissions"; import Checkbox from "@/components/Low/Checkbox"; import ReactDatePicker from "react-datepicker"; import clsx from "clsx"; -import countryCodes from "country-codes-list"; +import countryCodes, { CountryData } from "country-codes-list"; import { User, Type as UserType } from "@/interfaces/user"; import { Type, UserImport } from "../../../interfaces/IUserImport"; import UserTable from "../../../components/UserTable"; @@ -22,6 +21,7 @@ import Select from "@/components/Low/Select"; import { IoInformationCircleOutline } from "react-icons/io5"; import { FaFileDownload } from "react-icons/fa"; import { HiOutlineDocumentText } from "react-icons/hi"; +import UserImportSummary, { DuplicatesMap } from "@/components/UserImportSummary"; const EMAIL_REGEX = new RegExp(/^[a-zA-Z0-9]+(?:\.[a-zA-Z0-9]+)*@[a-zA-Z0-9]+(?:\.[a-zA-Z0-9]+)*$/); @@ -71,11 +71,14 @@ interface Props { onFinish: () => void; } + export default function BatchCreateUser({ user, entities = [], permissions, onFinish }: Props) { const [infos, setInfos] = useState([]); + const [parsedExcel, setParsedExcel] = useState<{ rows?: any[]; errors?: any[] }>({ rows: undefined, errors: undefined }); const [duplicatedUsers, setDuplicatedUsers] = useState([]); const [newUsers, setNewUsers] = useState([]); + const [duplicatedRows, setDuplicatedRows] = useState<{ duplicates: DuplicatesMap, count: number }>(); const [isLoading, setIsLoading] = useState(false); const [expiryDate, setExpiryDate] = useState( @@ -96,61 +99,187 @@ export default function BatchCreateUser({ user, entities = [], permissions, onFi if (!isExpiryDateEnabled) setExpiryDate(null); }, [isExpiryDateEnabled]); + const schema = { + 'First Name': { + prop: 'firstName', + type: String, + required: true, + validate: (value: string) => { + if (!value || value.trim() === '') { + throw new Error('First Name cannot be empty') + } + return true + } + }, + 'Last Name': { + prop: 'lastName', + type: String, + required: true, + validate: (value: string) => { + if (!value || value.trim() === '') { + throw new Error('Last Name cannot be empty') + } + return true + } + }, + 'Student ID': { + prop: 'studentID', + type: String, + required: true, + validate: (value: string) => { + if (!value || value.trim() === '') { + throw new Error('Student ID cannot be empty') + } + return true + } + }, + 'Passport/National ID': { + prop: 'passport_id', + type: String, + required: true, + validate: (value: string) => { + if (!value || value.trim() === '') { + throw new Error('Passport/National ID cannot be empty') + } + return true + } + }, + 'E-mail': { + prop: 'email', + required: true, + type: (value: any) => { + if (!value || value.trim() === '') { + throw new Error('Email cannot be empty') + } + if (!EMAIL_REGEX.test(value.trim())) { + throw new Error('Invalid Email') + } + return value + } + }, + 'Phone Number': { + prop: 'phone', + type: Number, + required: true, + validate: (value: number) => { + if (value === null || value === undefined || String(value).trim() === '') { + throw new Error('Phone Number cannot be empty') + } + return true + } + }, + 'Classroom Name': { + prop: 'group', + type: String, + required: true, + validate: (value: string) => { + if (!value || value.trim() === '') { + throw new Error('Classroom Name cannot be empty') + } + return true + } + }, + 'Country': { + prop: 'country', + type: (value: any) => { + if (!value || value.trim() === '') { + throw new Error('Country cannot be empty') + } + const validCountry = countryCodes.findOne("countryCode" as any, value.toUpperCase()) || + countryCodes.all().find((x) => x.countryNameEn.toLowerCase() === value.toLowerCase()); + if (!validCountry) { + throw new Error('Invalid Country/Country Code') + } + return validCountry; + }, + required: true + } + } + useEffect(() => { if (filesContent.length > 0) { const file = filesContent[0]; - readXlsxFile(file.content).then((rows) => { - try { - const information = uniqBy( - rows - .map((row) => { - const [firstName, lastName, studentID, passport_id, email, phone, group, country] = row as string[]; - const countryItem = - countryCodes.findOne("countryCode" as any, country.toUpperCase()) || - countryCodes.all().find((x) => x.countryNameEn.toLowerCase() === country.toLowerCase()); - - return EMAIL_REGEX.test(email.toString().trim()) - ? { - email: email.toString().trim().toLowerCase(), - name: `${firstName ?? ""} ${lastName ?? ""}`.trim(), - type: type, - passport_id: passport_id?.toString().trim() || undefined, - groupName: group, - studentID, - entity, - demographicInformation: { - country: countryItem?.countryCode, - passport_id: passport_id?.toString().trim() || undefined, - phone: phone.toString(), - }, - } - : undefined; - }) - .filter((x) => !!x) as typeof infos, - (x) => x.email, - ); - - if (information.length === 0) { - toast.error( - "Please upload an Excel file containing user information, one per line! All already registered e-mails have also been ignored!", - ); - return clear(); - } - - setInfos(information); - } catch (e) { - console.log(e) - - toast.error( - "Please upload an Excel file containing user information, one per line! All already registered e-mails have also been ignored!", - ); - return clear(); - } - }); + readXlsxFile( + file.content, { schema, ignoreEmptyRows: false }) + .then((data) => { + setParsedExcel(data) + }); } // eslint-disable-next-line react-hooks/exhaustive-deps }, [filesContent]); + useEffect(() => { + if (parsedExcel.rows) { + const duplicates: DuplicatesMap = { + studentID: new Map(), + email: new Map(), + passport_id: new Map(), + phone: new Map() + }; + const duplicateValues = new Set(); + const duplicateRowIndices = new Set(); + + const errorRowIndices = new Set( + parsedExcel.errors?.map(error => error.row) || [] + ); + + parsedExcel.rows.forEach((row, index) => { + if (!errorRowIndices.has(index + 2)) { + (Object.keys(duplicates) as Array).forEach(field => { + if (row !== null) { + const value = row[field]; + if (value) { + if (!duplicates[field].has(value)) { + duplicates[field].set(value, [index + 2]); + } else { + const existingRows = duplicates[field].get(value); + if (existingRows) { + existingRows.push(index + 2); + duplicateValues.add(value); + existingRows.forEach(rowNum => duplicateRowIndices.add(rowNum)); + } + } + } + } + }); + } + }); + + const infos = parsedExcel.rows + .map((row, index) => { + if (errorRowIndices.has(index + 2) || duplicateRowIndices.has(index + 2) || row === null) { + return undefined; + } + const { firstName, lastName, studentID, passport_id, email, phone, group, country } = row; + if (!email || !EMAIL_REGEX.test(email.toString().trim())) { + return undefined; + } + + return { + email: email.toString().trim().toLowerCase(), + name: `${firstName ?? ""} ${lastName ?? ""}`.trim(), + type: type, + passport_id: passport_id?.toString().trim() || undefined, + groupName: group, + studentID, + entity, + demographicInformation: { + country: country?.countryCode, + passport_id: passport_id?.toString().trim() || undefined, + phone: phone.toString(), + }, + } as UserImport; + }) + .filter((item): item is UserImport => item !== undefined); + + setDuplicatedRows({ duplicates, count: duplicateRowIndices.size }); + setInfos(infos); + setNewUsers([]); + setDuplicatedUsers([]); + } + }, [entity, parsedExcel, type]); + + useEffect(() => { const crossReferenceEmails = async () => { try { @@ -185,12 +314,12 @@ export default function BatchCreateUser({ user, entities = [], permissions, onFi if (!confirm(`You are about to ${[newUsersSentence, existingUsersSentence].filter((x) => !!x).join(" and ")}, are you sure you want to continue?`)) return; - Promise.all(newUsers.map(async (u) => await axios.post(`/api/invites`, {to: u.id, entity, from: user.id}))) + Promise.all(newUsers.map(async (u) => await axios.post(`/api/invites`, { to: u.id, entity, from: user.id }))) .then(() => toast.success(`Successfully invited ${newUsers.length} registered student(s)!`)) .finally(() => { if (newUsers.length === 0) setIsLoading(false); }); - + if (newUsers.length > 0) { setIsLoading(true); @@ -246,7 +375,7 @@ export default function BatchCreateUser({ user, entities = [], permissions, onFi be an Excel .xlsx document.
  • - only have a single spreadsheet with only the following 8 columns in the same order: + only have a single spreadsheet with the following exact same name columns:
    @@ -281,7 +410,7 @@ export default function BatchCreateUser({ user, entities = [], permissions, onFi all already registered e-mails will be ignored.
  • - the spreadsheet may have a header row with the format above, however, it is not necessary as long the columns are in the right order. + all rows which contain duplicate values in the columns: "Student ID", "Passport/National ID", "E-mail", will be ignored.
  • all of the e-mails in the file will receive an e-mail to join EnCoach with the role selected below. @@ -380,21 +509,16 @@ export default function BatchCreateUser({ user, entities = [], permissions, onFi ))} )} + {parsedExcel.rows !== undefined && } {newUsers.length !== 0 && (
    New Users:
    )} - {duplicatedUsers.length !== 0 && ( -
    - Duplicated Users: - -
    - )} {(duplicatedUsers.length !== 0 && newUsers.length === 0) && The imported .csv only contains duplicated users!} diff --git a/src/utils/excel.errors.ts b/src/utils/excel.errors.ts new file mode 100644 index 00000000..232441f7 --- /dev/null +++ b/src/utils/excel.errors.ts @@ -0,0 +1,27 @@ +export interface ExcelError { + type: string; + value: any; + error: string; + reason: string; + row: number; + column: string; +} + +export const errorsByRows = (errors: ExcelError[] ) => { + return errors.reduce((acc, error) => { + if (!acc[error.row]) { + acc[error.row] = { + required: [], + other: [] + }; + } + + if (error.error === 'required') { + acc[error.row].required.push(error.column); + } else { + acc[error.row].other.push(error); + } + + return acc; + }, {} as Record); +}