From 3348ca57983525554212dc077714c7526369eb30 Mon Sep 17 00:00:00 2001 From: Kavehhn174 Date: Sun, 16 Aug 2026 07:04:16 +0330 Subject: [PATCH] feat: import class sessions, attendance, and payments from JSON Match students by name when national ID or phone is missing, parse Jalali dates, and allow sessions without a professor. --- components/dataImport/dataImportService.js | 229 +++++++++++++++++--- components/dataImport/importHelpers.js | 84 +++++++ components/dataImport/importHelpers.test.js | 51 +++++ components/sessions/sessionModel.js | 1 - package.json | 2 +- utils/jalaliDate.js | 97 +++++++++ utils/jalaliDate.test.js | 18 ++ 7 files changed, 445 insertions(+), 37 deletions(-) create mode 100644 components/dataImport/importHelpers.js create mode 100644 components/dataImport/importHelpers.test.js create mode 100644 utils/jalaliDate.js create mode 100644 utils/jalaliDate.test.js diff --git a/components/dataImport/dataImportService.js b/components/dataImport/dataImportService.js index 8fe70ea..e82ba0e 100644 --- a/components/dataImport/dataImportService.js +++ b/components/dataImport/dataImportService.js @@ -6,27 +6,24 @@ const Course = require('../courses/courseModel'); const Class = require('../classes/classModel'); const User = require('../users/userModel'); const Role = require('../roles/roleModel'); +const Session = require('../sessions/sessionModel'); +const Payment = require('../payments/paymentModel'); const AppError = require('../../utils/AppError'); const { generateUsername, generateSimplePassword } = require('../../utils/credentials'); const { mergeFullName, normalizeGender } = require('../../utils/userProfile'); +const { parseImportDate, utcDayRange } = require('../../utils/jalaliDate'); +const { + namesMatch, + normalizePersonName, + normalizePhone, + normalizeNationalId, + mapAttendanceStatus +} = require('./importHelpers'); -const toEnglishDigits = (value) => - String(value ?? '') - .replace(/[۰-۹]/g, (d) => '۰۱۲۳۴۵۶۷۸۹'.indexOf(d)) - .replace(/[٠-٩]/g, (d) => '٠١٢٣٤٥٦٧٨٩'.indexOf(d)); +const ATTENDANCE_STATUSES = new Set(['present', 'absent', 'late', 'excused']); +const PAYMENT_METHODS = new Set(['online', 'card', 'cash']); -const normalizePhone = (raw) => { - if (raw == null || raw === '') return ''; - let digits = toEnglishDigits(raw).replace(/\D/g, ''); - if (digits.startsWith('98') && digits.length === 12) digits = `0${digits.slice(2)}`; - if (digits.length === 10 && digits.startsWith('9')) digits = `0${digits}`; - return digits; -}; - -const normalizeNationalId = (raw) => { - if (raw == null || raw === '') return ''; - return toEnglishDigits(raw).replace(/\D/g, ''); -}; +const escapeRegex = (value) => String(value).replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); const allocateUniqueUsername = async () => { for (let attempt = 0; attempt < 20; attempt += 1) { @@ -47,7 +44,7 @@ const allocatePlaceholderNationalId = async (phoneNumber) => { return candidate; }; -const findExistingUser = async ({ nationalIdCode, phoneNumber }) => { +const findExistingUser = async ({ nationalIdCode, phoneNumber, name }) => { if (nationalIdCode) { const byId = await User.findOne({ nationalIdCode }); if (byId) return byId; @@ -56,6 +53,15 @@ const findExistingUser = async ({ nationalIdCode, phoneNumber }) => { const byPhone = await User.findOne({ phoneNumber }); if (byPhone) return byPhone; } + const target = normalizePersonName(name); + if (target) { + const firstToken = target.split(' ')[0]; + const candidates = await User.find({ + name: new RegExp(escapeRegex(firstToken), 'i') + }).limit(50); + const match = candidates.find((user) => namesMatch(user.name, target)); + if (match) return match; + } return null; }; @@ -100,7 +106,7 @@ const upsertStudent = async (student, userRole, stats, warnings) => { return null; } - let user = await findExistingUser({ nationalIdCode, phoneNumber }); + let user = await findExistingUser({ nationalIdCode, phoneNumber, name: student.name }); if (user) { applyStudentProfile(user, student); if (phoneNumber && !user.phoneNumber) user.phoneNumber = phoneNumber; @@ -126,7 +132,6 @@ const upsertStudent = async (student, userRole, stats, warnings) => { }); } - // Collision: national id exists with different phone, or reverse const conflictById = await User.findOne({ nationalIdCode }); if (conflictById) { applyStudentProfile(conflictById, student); @@ -164,6 +169,159 @@ const upsertStudent = async (student, userRole, stats, warnings) => { return user; }; +const enrollUserInClass = async (user, course, cls, stats) => { + if (!user.courses.map(String).includes(String(course._id))) { + user.courses.push(course._id); + await user.save(); + } + if (!cls.students.map(String).includes(String(user._id))) { + cls.students.push(user._id); + stats.enrollmentsAdded += 1; + } +}; + +const applyClassSchedule = (cls, classInput) => { + if (classInput.startDate && !cls.startDate) { + const startDate = parseImportDate(classInput.startDate); + if (startDate) cls.startDate = startDate; + } + if (Array.isArray(classInput.days) && classInput.days.length) { + cls.days = classInput.days; + } + if (classInput.startTime) cls.startTime = String(classInput.startTime).trim(); + if (classInput.endTime) cls.endTime = String(classInput.endTime).trim(); + if (classInput.tuitionFee != null && !cls.tuitionFee) { + cls.tuitionFee = Number(classInput.tuitionFee) || 0; + } +}; + +const resolveLinkedUser = async (record, warnings, reason, statsSkipKey, stats) => { + const phoneNumber = normalizePhone(record.phoneNumber || record.phone); + const nationalIdCode = normalizeNationalId(record.nationalIdCode || record.nationalId); + const user = await findExistingUser({ + nationalIdCode, + phoneNumber, + name: record.name + }); + if (user) return user; + warnings.push({ + reason, + student: { name: record.name, phoneNumber, nationalIdCode } + }); + stats[statsSkipKey] += 1; + return null; +}; + +const importSessions = async (classInput, course, cls, stats, warnings) => { + const sessions = Array.isArray(classInput.sessions) ? classInput.sessions : []; + const defaultStart = classInput.startTime || cls.startTime || '19:00'; + const defaultEnd = classInput.endTime || cls.endTime || '20:30'; + + for (const sessionInput of sessions) { + const range = utcDayRange(sessionInput.day || sessionInput.date); + if (!range) { + warnings.push({ reason: 'invalid_session_date', session: { topic: sessionInput.topic, day: sessionInput.day } }); + continue; + } + + const attendanceSource = Array.isArray(sessionInput.attendance) + ? sessionInput.attendance + : (sessionInput.attendanceList || []); + const attendanceList = []; + + for (const record of attendanceSource) { + const mapped = mapAttendanceStatus(record.status) || String(record.status || '').trim(); + if (!ATTENDANCE_STATUSES.has(mapped)) continue; + const user = await resolveLinkedUser(record, warnings, 'attendance_user_not_found', 'attendanceSkipped', stats); + if (!user) continue; + await enrollUserInClass(user, course, cls, stats); + attendanceList.push({ + user: user._id, + status: mapped, + note: record.note || '' + }); + stats.attendanceRecords += 1; + } + + const payload = { + course: course._id, + class: cls._id, + day: range.start, + startTime: sessionInput.startTime || defaultStart, + endTime: sessionInput.endTime || defaultEnd, + topic: sessionInput.topic || '', + place: sessionInput.place || '', + note: sessionInput.note || '', + status: sessionInput.status || (attendanceList.length ? 'held' : 'scheduled'), + attendanceList + }; + if (cls.professor) payload.professor = cls.professor; + + let session = await Session.findOne({ + class: cls._id, + day: { $gte: range.start, $lt: range.end } + }); + if (!session) { + await Session.create(payload); + stats.sessionsCreated += 1; + } else { + Object.assign(session, payload); + await session.save(); + stats.sessionsReused += 1; + } + } +}; + +const importPayments = async (classInput, course, cls, stats, warnings) => { + const payments = Array.isArray(classInput.payments) ? classInput.payments : []; + + for (const paymentInput of payments) { + const amount = Number(paymentInput.amount) || 0; + if (!amount) { + stats.paymentsSkipped += 1; + continue; + } + + const user = await resolveLinkedUser(paymentInput, warnings, 'payment_user_not_found', 'paymentsSkipped', stats); + if (!user) continue; + await enrollUserInClass(user, course, cls, stats); + + const transactions = (Array.isArray(paymentInput.transactions) ? paymentInput.transactions : []) + .map((trx) => ({ + amount: Number(trx.amount) || 0, + method: PAYMENT_METHODS.has(trx.method) ? trx.method : 'card', + receiptNumber: trx.receiptNumber != null && trx.receiptNumber !== '' ? String(trx.receiptNumber) : '', + notes: trx.notes || '', + date: parseImportDate(trx.date) || new Date() + })) + .filter((trx) => trx.amount > 0); + + const payload = { + user: user._id, + classes: [cls._id], + course: course._id, + amount, + discount: Number(paymentInput.discount) || 0, + paidAmount: transactions.reduce((sum, trx) => sum + trx.amount, 0), + transactions, + notes: paymentInput.notes || '', + dueDate: parseImportDate(paymentInput.dueDate) || undefined + }; + + const existing = await Payment.findOne({ user: user._id, classes: cls._id }); + if (!existing) { + await Payment.create(payload); + stats.paymentsCreated += 1; + } else if (!existing.transactions?.length) { + Object.assign(existing, payload); + await existing.save(); + stats.paymentsUpdated += 1; + } else { + stats.paymentsReused += 1; + } + } +}; + const importData = async (payload) => { if (!payload || !Array.isArray(payload.courses)) { throw new AppError('VALIDATION_FAILED', null, 'Invalid import payload: courses array required'); @@ -180,7 +338,15 @@ const importData = async (payload) => { studentsCreated: 0, studentsUpdated: 0, studentsSkipped: 0, - enrollmentsAdded: 0 + enrollmentsAdded: 0, + sessionsCreated: 0, + sessionsReused: 0, + attendanceRecords: 0, + attendanceSkipped: 0, + paymentsCreated: 0, + paymentsUpdated: 0, + paymentsReused: 0, + paymentsSkipped: 0 }; const warnings = []; @@ -217,35 +383,28 @@ const importData = async (payload) => { cls = await Class.create({ name: className, course: course._id, - startDate: classInput.startDate ? new Date(classInput.startDate) : undefined, + startDate: parseImportDate(classInput.startDate) || undefined, tuitionFee: Number(classInput.tuitionFee) || Number(courseInput.price) || 0, + days: Array.isArray(classInput.days) ? classInput.days : [], + startTime: classInput.startTime || '', + endTime: classInput.endTime || '', isActive: true }); stats.classesCreated += 1; } else { stats.classesReused += 1; - if (!cls.startDate && classInput.startDate) { - cls.startDate = new Date(classInput.startDate); - await cls.save(); - } + applyClassSchedule(cls, classInput); } const students = Array.isArray(classInput.students) ? classInput.students : []; for (const student of students) { const user = await upsertStudent(student, userRole, stats, warnings); if (!user) continue; - - if (!user.courses.map(String).includes(String(course._id))) { - user.courses.push(course._id); - await user.save(); - } - - if (!cls.students.map(String).includes(String(user._id))) { - cls.students.push(user._id); - stats.enrollmentsAdded += 1; - } + await enrollUserInClass(user, course, cls, stats); } + await importSessions(classInput, course, cls, stats, warnings); + await importPayments(classInput, course, cls, stats, warnings); await cls.save(); } } diff --git a/components/dataImport/importHelpers.js b/components/dataImport/importHelpers.js new file mode 100644 index 0000000..a515872 --- /dev/null +++ b/components/dataImport/importHelpers.js @@ -0,0 +1,84 @@ +'use strict'; + +const { toEnglishDigits } = require('../../utils/jalaliDate'); + +const TITLE_PREFIX = /^(آقای|اقای|خانم|آقا)\s+/; +const HONORIFIC_MIDDLE = /\s+خ\s+/g; + +const normalizePersonName = (value) => { + const text = String(value ?? '') + .replace(/ي/g, 'ی') + .replace(/ك/g, 'ک') + .replace(/[\u200c\u200d]/g, ' ') + .replace(/\s+/g, ' ') + .trim() + .replace(TITLE_PREFIX, '') + .replace(HONORIFIC_MIDDLE, ' ') + .replace(/^خ\s+/, '') + .trim(); + return text; +}; + +const namesMatch = (left, right) => { + const a = normalizePersonName(left); + const b = normalizePersonName(right); + return Boolean(a) && a === b; +}; + +const normalizePhone = (raw) => { + if (raw == null || raw === '') return ''; + let digits = toEnglishDigits(raw).replace(/\D/g, ''); + if (digits.startsWith('98') && digits.length === 12) digits = `0${digits.slice(2)}`; + if (digits.length === 10 && digits.startsWith('9')) digits = `0${digits}`; + return digits; +}; + +const normalizeNationalId = (raw) => { + if (raw == null || raw === '') return ''; + return toEnglishDigits(raw).replace(/\D/g, ''); +}; + +const mapAttendanceStatus = (raw) => { + const value = String(raw ?? '').trim(); + if (!value) return null; + if (value === '*' || value === 'ح' || value === 'حاضر' || /^present$/i.test(value) || /^p$/i.test(value)) { + return 'present'; + } + if (value === 'غ' || value === 'غایب' || /^absent$/i.test(value) || /^a$/i.test(value)) { + return 'absent'; + } + if (value === 'ت' || value === 'تأخیر' || value === 'تاخیر' || /^late$/i.test(value)) { + return 'late'; + } + if (value === 'م' || value === 'موجه' || /^excused$/i.test(value)) { + return 'excused'; + } + return null; +}; + +const projectSessionDates = (existingIsoDates, totalCount) => { + const dates = [...existingIsoDates].filter(Boolean).sort(); + if (!dates.length || dates.length >= totalCount) return dates.slice(0, totalCount); + + const weekdaySet = new Set(dates.map((iso) => new Date(`${iso}T12:00:00.000Z`).getUTCDay())); + const last = new Date(`${dates[dates.length - 1]}T12:00:00.000Z`); + const projected = [...dates]; + const cursor = new Date(last); + + while (projected.length < totalCount) { + cursor.setUTCDate(cursor.getUTCDate() + 1); + if (weekdaySet.has(cursor.getUTCDay())) { + projected.push(cursor.toISOString().slice(0, 10)); + } + } + return projected; +}; + +module.exports = { + normalizePersonName, + namesMatch, + normalizePhone, + normalizeNationalId, + mapAttendanceStatus, + projectSessionDates +}; diff --git a/components/dataImport/importHelpers.test.js b/components/dataImport/importHelpers.test.js new file mode 100644 index 0000000..37b8f96 --- /dev/null +++ b/components/dataImport/importHelpers.test.js @@ -0,0 +1,51 @@ +'use strict'; + +const { describe, it } = require('node:test'); +const assert = require('node:assert/strict'); +const { + normalizePersonName, + namesMatch, + normalizePhone, + mapAttendanceStatus, + projectSessionDates +} = require('./importHelpers'); + +describe('importHelpers', () => { + it('normalizes honorifics so spreadsheet names match database users', () => { + assert.equal(normalizePersonName('مریم خ پناهیان'), 'مریم پناهیان'); + assert.ok(namesMatch('آقای رضایی', 'آقای رضایی')); + assert.equal(normalizePhone(9168942141), '09168942141'); + }); + + it('maps spreadsheet attendance marks', () => { + assert.equal(mapAttendanceStatus('*'), 'present'); + assert.equal(mapAttendanceStatus('غ'), 'absent'); + assert.equal(mapAttendanceStatus(''), null); + }); + + it('projects remaining Teksa sessions from Sunday/Tuesday pattern', () => { + const firstNine = [ + '2026-07-14', + '2026-07-19', + '2026-07-21', + '2026-07-26', + '2026-07-28', + '2026-08-02', + '2026-08-04', + '2026-08-09', + '2026-08-11' + ]; + const all = projectSessionDates(firstNine, 16); + assert.equal(all.length, 16); + assert.deepEqual(all.slice(0, 9), firstNine); + assert.deepEqual(all.slice(9), [ + '2026-08-16', + '2026-08-18', + '2026-08-23', + '2026-08-25', + '2026-08-30', + '2026-09-01', + '2026-09-06' + ]); + }); +}); diff --git a/components/sessions/sessionModel.js b/components/sessions/sessionModel.js index af8819c..8c878fb 100644 --- a/components/sessions/sessionModel.js +++ b/components/sessions/sessionModel.js @@ -41,7 +41,6 @@ const sessionSchema = new mongoose.Schema({ professor: { type: mongoose.Schema.Types.ObjectId, ref: 'Professor', - required: true, index: true }, day: { diff --git a/package.json b/package.json index 33c522a..b68db1f 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,7 @@ "start": "node app.js", "dev": "nodemon app.js", "seed": "node seed.js", - "test": "node --test components/settings/smsTemplates.test.js components/users/passwordReset.test.js utils/classSchedule.test.js utils/messagingChannels.test.js utils/notifyFlags.test.js utils/passwordRules.test.js utils/paymentAmount.test.js" + "test": "node --test components/settings/smsTemplates.test.js components/users/passwordReset.test.js components/dataImport/importHelpers.test.js utils/classSchedule.test.js utils/jalaliDate.test.js utils/messagingChannels.test.js utils/notifyFlags.test.js utils/passwordRules.test.js utils/paymentAmount.test.js" }, "keywords": [ "express", diff --git a/utils/jalaliDate.js b/utils/jalaliDate.js new file mode 100644 index 0000000..70c20c4 --- /dev/null +++ b/utils/jalaliDate.js @@ -0,0 +1,97 @@ +'use strict'; + +const toEnglishDigits = (value) => + String(value ?? '') + .replace(/[۰-۹]/g, (d) => '۰۱۲۳۴۵۶۷۸۹'.indexOf(d)) + .replace(/[٠-٩]/g, (d) => '٠١٢٣٤٥٦٧٨٩'.indexOf(d)); + +const jalaliToGregorian = (jy, jm, jd) => { + const gy = jy <= 979 ? 621 : 1600; + jy -= jy <= 979 ? 0 : 979; + let days = + 365 * jy + + Math.floor(jy / 33) * 8 + + Math.floor(((jy % 33) + 3) / 4) + + 78 + + jd + + (jm < 7 ? (jm - 1) * 31 : (jm - 7) * 30 + 186); + let gyOut = gy + 400 * Math.floor(days / 146097); + days %= 146097; + if (days > 36524) { + gyOut += 100 * Math.floor(--days / 36524); + days %= 36524; + if (days >= 365) days += 1; + } + gyOut += 4 * Math.floor(days / 1461); + days %= 1461; + if (days > 365) { + gyOut += Math.floor((days - 1) / 365); + days = (days - 1) % 365; + } + let gd = days + 1; + const sal_a = [ + 0, + 31, + (gyOut % 4 === 0 && gyOut % 100 !== 0) || gyOut % 400 === 0 ? 29 : 28, + 31, + 30, + 31, + 30, + 31, + 31, + 30, + 31, + 30, + 31 + ]; + let gm = 0; + for (gm = 1; gm <= 12 && gd > sal_a[gm]; gm += 1) gd -= sal_a[gm]; + return `${gyOut}-${String(gm).padStart(2, '0')}-${String(gd).padStart(2, '0')}`; +}; + +const parseJalaliDate = (raw) => { + if (!raw) return null; + const text = toEnglishDigits(raw).replace(/[./\-]/g, '/').trim(); + const match = text.match(/^(\d{3,4})\/(\d{1,2})\/(\d{1,2})$/); + if (!match) return null; + const jy = Number(match[1]); + const jm = Number(match[2]); + const jd = Number(match[3]); + if (!jy || !jm || !jd || jm > 12 || jd > 31 || jy < 1200 || jy > 1599) return null; + try { + return jalaliToGregorian(jy, jm, jd); + } catch { + return null; + } +}; + +const parseImportDate = (raw) => { + if (!raw) return null; + if (raw instanceof Date) { + return Number.isNaN(raw.getTime()) ? null : raw; + } + const text = String(raw).trim(); + if (!text) return null; + const gregorian = text.match(/^(\d{4}-\d{2}-\d{2})/); + if (gregorian) return new Date(`${gregorian[1]}T00:00:00.000Z`); + const jalali = parseJalaliDate(text); + if (jalali) return new Date(`${jalali}T00:00:00.000Z`); + const date = new Date(text); + return Number.isNaN(date.getTime()) ? null : date; +}; + +const utcDayRange = (value) => { + const date = parseImportDate(value); + if (!date) return null; + const start = new Date(Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate())); + const end = new Date(start.getTime() + 24 * 60 * 60 * 1000); + return { start, end }; +}; + +module.exports = { + toEnglishDigits, + jalaliToGregorian, + parseJalaliDate, + parseImportDate, + utcDayRange +}; diff --git a/utils/jalaliDate.test.js b/utils/jalaliDate.test.js new file mode 100644 index 0000000..8c346f1 --- /dev/null +++ b/utils/jalaliDate.test.js @@ -0,0 +1,18 @@ +'use strict'; + +const { describe, it } = require('node:test'); +const assert = require('node:assert/strict'); +const { jalaliToGregorian, parseJalaliDate, parseImportDate } = require('./jalaliDate'); + +describe('jalaliDate', () => { + it('converts تکسا attendance dates to Gregorian', () => { + assert.equal(jalaliToGregorian(1405, 4, 23), '2026-07-14'); + assert.equal(parseJalaliDate('1405.4.28'), '2026-07-19'); + assert.equal(parseJalaliDate('1405/5/20'), '2026-08-11'); + }); + + it('parses ISO and jalali values into UTC midnight dates', () => { + assert.equal(parseImportDate('2026-07-14').toISOString(), '2026-07-14T00:00:00.000Z'); + assert.equal(parseImportDate('1405.5.12').toISOString(), '2026-08-03T00:00:00.000Z'); + }); +});