feat: add append-only course/user data import and expand user profile

Support uploading structured JSON to create courses, classes, and students without wiping existing data, and merge user name fields while adding gender and registration details from source sheets.

components/activityLogs/activityLogService.js

@@ -121,7 +121,7 @@ const getAllActivityLogs = async (queryParams = {}) => {
 
   const [logs, totalCount] = await Promise.all([
     ActivityLog.find(filter)
-      .populate('actor', 'name surname username')
+      .populate('actor', 'name username')
       .sort(sort)
       .skip(skip)
       .limit(limit)

components/certificates/certificateService.js

@@ -39,7 +39,7 @@ const createCertificate = async (data) => {
 
 const getCertificateById = async (id) => {
   const certificate = await Certificate.findById(id)
-    .populate('user', 'name surname username nationalIdCode')
+    .populate('user', 'name username nationalIdCode')
     .populate('course', 'title type');
   if (!certificate) {
     throw new AppError('CERTIFICATE_NOT_FOUND');
@@ -56,7 +56,7 @@ const getAllCertificates = async (queryParams) => {
 
   const [certificates, totalCount] = await Promise.all([
     Certificate.find(filter)
-      .populate('user', 'name surname username')
+      .populate('user', 'name username')
       .populate('course', 'title')
       .sort(sort)
       .skip(skip)

components/classes/classService.js

@@ -32,7 +32,7 @@ const getOne = async (id) => {
   const cls = await Class.findById(id)
     .populate({ path: 'course', select: 'title type price' })
     .populate({ path: 'professor', select: 'name surname phoneNumber' })
-    .populate({ path: 'students', select: 'name surname phoneNumber' })
+      .populate({ path: 'students', select: 'name phoneNumber gender' })
     .lean();
   if (!cls) throw new AppError('CLASS_NOT_FOUND');
   return cls;

components/dataImport/dataImportController.js

@@ -0,0 +1,28 @@
+// /components/dataImport/dataImportController.js
+'use strict';
+
+const catchAsync = require('../../utils/catchAsync');
+const dataImportService = require('./dataImportService');
+const AppError = require('../../utils/AppError');
+const { successResponse } = require('../../utils/apiResponse');
+
+exports.importJson = catchAsync(async (req, res) => {
+  let payload = req.body;
+
+  if (req.file?.buffer) {
+    try {
+      payload = JSON.parse(req.file.buffer.toString('utf8'));
+    } catch {
+      throw new AppError('VALIDATION_FAILED', null, 'Uploaded file is not valid JSON');
+    }
+  }
+
+  // Allow { data: { courses: [...] } } wrappers
+  if (payload?.data?.courses) payload = payload.data;
+  if (!payload?.courses && payload?.version && payload?.courses == null) {
+    throw new AppError('VALIDATION_FAILED', null, 'Invalid import payload');
+  }
+
+  const result = await dataImportService.importData(payload);
+  return successResponse(res, 200, 'Data imported successfully', result);
+});

components/dataImport/dataImportRoutes.js

@@ -0,0 +1,21 @@
+// /components/dataImport/dataImportRoutes.js
+'use strict';
+
+const express = require('express');
+const multer = require('multer');
+const dataImportController = require('./dataImportController');
+const authMiddleware = require('../../middlewares/authMiddleware');
+const requireSuperAdmin = require('../../middlewares/requireSuperAdmin');
+
+const router = express.Router();
+const upload = multer({
+  storage: multer.memoryStorage(),
+  limits: { fileSize: 20 * 1024 * 1024 }
+});
+
+router.use(authMiddleware);
+router.use(requireSuperAdmin);
+
+router.post('/', upload.single('file'), dataImportController.importJson);
+
+module.exports = router;

components/dataImport/dataImportService.js

@@ -0,0 +1,258 @@
+// /components/dataImport/dataImportService.js
+'use strict';
+
+const bcrypt = require('bcryptjs');
+const Course = require('../courses/courseModel');
+const Class = require('../classes/classModel');
+const User = require('../users/userModel');
+const Role = require('../roles/roleModel');
+const AppError = require('../../utils/AppError');
+const { generateUsername, generateSimplePassword } = require('../../utils/credentials');
+const { mergeFullName, normalizeGender } = require('../../utils/userProfile');
+
+const toEnglishDigits = (value) =>
+  String(value ?? '')
+    .replace(/[۰-۹]/g, (d) => '۰۱۲۳۴۵۶۷۸۹'.indexOf(d))
+    .replace(/[٠-٩]/g, (d) => '٠١٢٣٤٥٦٧٨٩'.indexOf(d));
+
+const normalizePhone = (raw) => {
+  if (raw == null || raw === '') return '';
+  let digits = toEnglishDigits(raw).replace(/\D/g, '');
+  if (digits.startsWith('98') && digits.length === 12) digits = `0${digits.slice(2)}`;
+  if (digits.length === 10 && digits.startsWith('9')) digits = `0${digits}`;
+  return digits;
+};
+
+const normalizeNationalId = (raw) => {
+  if (raw == null || raw === '') return '';
+  return toEnglishDigits(raw).replace(/\D/g, '');
+};
+
+const allocateUniqueUsername = async () => {
+  for (let attempt = 0; attempt < 20; attempt += 1) {
+    const username = generateUsername();
+    if (!(await User.exists({ username }))) return username;
+  }
+  throw new AppError('INTERNAL_SERVER_ERROR', null, 'Could not generate a unique username');
+};
+
+const allocatePlaceholderNationalId = async (phoneNumber) => {
+  const base = `TMP${(phoneNumber || '').replace(/\D/g, '').slice(-10) || Date.now().toString().slice(-10)}`;
+  let candidate = base.padEnd(10, '0').slice(0, 10);
+  let i = 0;
+  while (await User.exists({ nationalIdCode: candidate })) {
+    i += 1;
+    candidate = `${base.slice(0, 7)}${String(i).padStart(3, '0')}`.slice(0, 10);
+  }
+  return candidate;
+};
+
+const findExistingUser = async ({ nationalIdCode, phoneNumber }) => {
+  if (nationalIdCode) {
+    const byId = await User.findOne({ nationalIdCode });
+    if (byId) return byId;
+  }
+  if (phoneNumber) {
+    const byPhone = await User.findOne({ phoneNumber });
+    if (byPhone) return byPhone;
+  }
+  return null;
+};
+
+const applyStudentProfile = (user, student) => {
+  const name = mergeFullName(student.name, student.surname);
+  if (name) user.name = name;
+
+  const gender = normalizeGender(student.gender);
+  if (gender) user.gender = gender;
+
+  const optionalFields = [
+    'address',
+    'birthCertificateNumber',
+    'postalCode',
+    'placeOfIssue',
+    'fatherName',
+    'education',
+    'parentPhoneNumber'
+  ];
+  for (const field of optionalFields) {
+    if (student[field] && !user[field]) {
+      user[field] = String(student[field]).trim();
+    }
+  }
+
+  if (student.birthDate && !user.birthDate) {
+    const date = new Date(student.birthDate);
+    if (!Number.isNaN(date.getTime())) user.birthDate = date;
+  }
+};
+
+const upsertStudent = async (student, userRole, stats, warnings) => {
+  const phoneNumber = normalizePhone(student.phoneNumber || student.phone || student.parentPhoneNumber);
+  let nationalIdCode = normalizeNationalId(student.nationalIdCode || student.nationalId);
+
+  if (!phoneNumber && !nationalIdCode) {
+    warnings.push({
+      reason: 'missing_identity',
+      student: { name: student.name }
+    });
+    stats.studentsSkipped += 1;
+    return null;
+  }
+
+  let user = await findExistingUser({ nationalIdCode, phoneNumber });
+  if (user) {
+    applyStudentProfile(user, student);
+    if (phoneNumber && !user.phoneNumber) user.phoneNumber = phoneNumber;
+    await user.save();
+    stats.studentsUpdated += 1;
+    return user;
+  }
+
+  if (!phoneNumber) {
+    warnings.push({
+      reason: 'missing_phone',
+      student: { name: student.name, nationalIdCode }
+    });
+    stats.studentsSkipped += 1;
+    return null;
+  }
+
+  if (!nationalIdCode) {
+    nationalIdCode = await allocatePlaceholderNationalId(phoneNumber);
+    warnings.push({
+      reason: 'placeholder_national_id',
+      student: { name: student.name, phoneNumber, nationalIdCode }
+    });
+  }
+
+  // Collision: national id exists with different phone, or reverse
+  const conflictById = await User.findOne({ nationalIdCode });
+  if (conflictById) {
+    applyStudentProfile(conflictById, student);
+    await conflictById.save();
+    stats.studentsUpdated += 1;
+    return conflictById;
+  }
+
+  const username = await allocateUniqueUsername();
+  const plainPassword = generateSimplePassword();
+  const passwordHash = await bcrypt.hash(plainPassword, 10);
+
+  user = await User.create({
+    name: mergeFullName(student.name, student.surname) || `کاربر ${phoneNumber}`,
+    gender: normalizeGender(student.gender),
+    nationalIdCode,
+    phoneNumber,
+    address: student.address,
+    birthCertificateNumber: student.birthCertificateNumber,
+    postalCode: student.postalCode,
+    placeOfIssue: student.placeOfIssue,
+    fatherName: student.fatherName,
+    birthDate: student.birthDate ? new Date(student.birthDate) : undefined,
+    education: student.education,
+    parentPhoneNumber: student.parentPhoneNumber
+      ? normalizePhone(student.parentPhoneNumber)
+      : undefined,
+    username,
+    passwordHash,
+    role: userRole._id,
+    preferredMessenger: ['SMS']
+  });
+
+  stats.studentsCreated += 1;
+  return user;
+};
+
+const importData = async (payload) => {
+  if (!payload || !Array.isArray(payload.courses)) {
+    throw new AppError('VALIDATION_FAILED', null, 'Invalid import payload: courses array required');
+  }
+
+  const userRole = await Role.findOne({ name: 'User' });
+  if (!userRole) throw new AppError('DEFAULT_ROLE_NOT_FOUND');
+
+  const stats = {
+    coursesCreated: 0,
+    coursesReused: 0,
+    classesCreated: 0,
+    classesReused: 0,
+    studentsCreated: 0,
+    studentsUpdated: 0,
+    studentsSkipped: 0,
+    enrollmentsAdded: 0
+  };
+  const warnings = [];
+
+  for (const courseInput of payload.courses) {
+    const title = String(courseInput.title || '').trim();
+    if (!title) continue;
+
+    const type = courseInput.type === 'Private' ? 'Private' : 'General';
+    let course = await Course.findOne({ title });
+    if (!course) {
+      course = await Course.create({
+        title,
+        type,
+        price: Number(courseInput.price) || 0,
+        showOnFrontend: false,
+        description: courseInput.description || ''
+      });
+      stats.coursesCreated += 1;
+    } else {
+      stats.coursesReused += 1;
+      if (type === 'Private' && course.type !== 'Private') {
+        course.type = 'Private';
+        await course.save();
+      }
+    }
+
+    const classes = Array.isArray(courseInput.classes) ? courseInput.classes : [];
+    for (const classInput of classes) {
+      const className = String(classInput.name || '').trim();
+      if (!className) continue;
+
+      let cls = await Class.findOne({ name: className, course: course._id });
+      if (!cls) {
+        cls = await Class.create({
+          name: className,
+          course: course._id,
+          startDate: classInput.startDate ? new Date(classInput.startDate) : undefined,
+          tuitionFee: Number(classInput.tuitionFee) || Number(courseInput.price) || 0,
+          isActive: true
+        });
+        stats.classesCreated += 1;
+      } else {
+        stats.classesReused += 1;
+        if (!cls.startDate && classInput.startDate) {
+          cls.startDate = new Date(classInput.startDate);
+          await cls.save();
+        }
+      }
+
+      const students = Array.isArray(classInput.students) ? classInput.students : [];
+      for (const student of students) {
+        const user = await upsertStudent(student, userRole, stats, warnings);
+        if (!user) continue;
+
+        if (!user.courses.map(String).includes(String(course._id))) {
+          user.courses.push(course._id);
+          await user.save();
+        }
+
+        if (!cls.students.map(String).includes(String(user._id))) {
+          cls.students.push(user._id);
+          stats.enrollmentsAdded += 1;
+        }
+      }
+
+      await cls.save();
+    }
+  }
+
+  return { stats, warnings };
+};
+
+module.exports = {
+  importData
+};

components/notifications/notificationService.js

@@ -37,7 +37,7 @@ const createNotification = async (data) => {
 };
 
 const getNotificationById = async (id) => {
-  const notification = await Notification.findById(id).populate('user', 'name surname username email phoneNumber');
+  const notification = await Notification.findById(id).populate('user', 'name username email phoneNumber');
   if (!notification) {
     throw new AppError('NOTIFICATION_NOT_FOUND');
   }
@@ -49,7 +49,7 @@ const getAllNotifications = async (queryParams) => {
   const filter = buildFilterQuery(queryParams, ['subject', 'body']);
 
   const [notifications, totalCount] = await Promise.all([
-    Notification.find(filter).populate('user', 'name surname username').sort(sort).skip(skip).limit(limit),
+    Notification.find(filter).populate('user', 'name username').sort(sort).skip(skip).limit(limit),
     Notification.countDocuments(filter)
   ]);
 

components/payments/paymentService.js

@@ -18,7 +18,7 @@ const getAllPayments = async (query) => {
 
   const [items, total] = await Promise.all([
     Payment.find(filter)
-      .populate({ path: 'user', select: 'name surname' })
+      .populate({ path: 'user', select: 'name' })
       .populate({ path: 'classes', select: 'name' })
       .populate({ path: 'course', select: 'title' })
       .skip(skip).limit(limit).sort({ createdAt: -1 }).lean(),
@@ -30,7 +30,7 @@ const getAllPayments = async (query) => {
 
 const getPaymentById = async (id) => {
   const payment = await Payment.findById(id)
-    .populate({ path: 'user', select: 'name surname phoneNumber' })
+    .populate({ path: 'user', select: 'name phoneNumber' })
     .populate({ path: 'classes', select: 'name tuitionFee' })
     .populate({ path: 'course', select: 'title price' })
     .lean();

components/sessions/sessionService.js

@@ -85,10 +85,10 @@ const getSessionById = async (id) => {
     .populate({
       path: 'class',
       select: 'name students capacity',
-      populate: { path: 'students', select: 'name surname nationalIdCode phoneNumber' }
+      populate: { path: 'students', select: 'name nationalIdCode phoneNumber gender' }
     })
     .populate('professor', 'name surname email phoneNumber')
-    .populate('attendanceList.user', 'name surname username nationalIdCode');
+    .populate('attendanceList.user', 'name username nationalIdCode');
   if (!session) {
     throw new AppError('SESSION_NOT_FOUND');
   }

components/users/userModel.js

@@ -21,9 +21,9 @@ const userSchema = new mongoose.Schema({
     required: true,
     trim: true
   },
-  surname: {
+  gender: {
     type: String,
-    required: true,
+    enum: ['male', 'female'],
     trim: true
   },
   phoneNumber: {
@@ -51,6 +51,33 @@ const userSchema = new mongoose.Schema({
     type: String,
     trim: true
   },
+  birthCertificateNumber: {
+    type: String,
+    trim: true
+  },
+  postalCode: {
+    type: String,
+    trim: true
+  },
+  placeOfIssue: {
+    type: String,
+    trim: true
+  },
+  fatherName: {
+    type: String,
+    trim: true
+  },
+  birthDate: {
+    type: Date
+  },
+  education: {
+    type: String,
+    trim: true
+  },
+  parentPhoneNumber: {
+    type: String,
+    trim: true
+  },
   username: {
     type: String,
     required: true,

components/users/userService.js

@@ -9,6 +9,7 @@ const { calculateMeta } = require('../../utils/pagination');
 const { generateUsername, generateSimplePassword } = require('../../utils/credentials');
 const { sendAccountCreatedSms } = require('../../utils/senders/smsMessages');
 const logger = require('../../utils/logger');
+const { mergeFullName, normalizeGender } = require('../../utils/userProfile');
 
 const POPULATE_ROLE = { path: 'role', select: 'name permissions' };
 const SAFE_FIELDS = '-passwordHash -refreshTokens';
@@ -37,23 +38,46 @@ const allocateUniqueUsername = async (preferred) => {
   throw new AppError('INTERNAL_SERVER_ERROR', null, 'Could not generate a unique username');
 };
 
+const pickProfileFields = (body = {}) => {
+  const fullName = mergeFullName(body.name, body.surname);
+  const fields = {
+    name: fullName || undefined,
+    gender: normalizeGender(body.gender),
+    nationalIdCode: body.nationalIdCode || body.nationalId || undefined,
+    phoneNumber: body.phoneNumber || body.phone || undefined,
+    email: body.email,
+    address: body.address,
+    birthCertificateNumber: body.birthCertificateNumber,
+    postalCode: body.postalCode,
+    placeOfIssue: body.placeOfIssue,
+    fatherName: body.fatherName,
+    birthDate: body.birthDate || undefined,
+    education: body.education,
+    parentPhoneNumber: body.parentPhoneNumber,
+    preferredMessenger: normalizePreferredMessengers(body.preferredMessenger)
+  };
+
+  Object.keys(fields).forEach((key) => {
+    if (fields[key] === '' || fields[key] === null || fields[key] === undefined) {
+      delete fields[key];
+    }
+  });
+
+  return fields;
+};
+
 const signUp = async (body) => {
-  const { name, surname, nationalId, nationalIdCode, phoneNumber, phone, username, password, email, address, preferredMessenger } = body;
-  
+  const profile = pickProfileFields(body);
+  const { username, password } = body;
+
   const userRole = await Role.findOne({ name: 'User' });
   if (!userRole) throw new AppError('DEFAULT_ROLE_NOT_FOUND');
 
   const passwordHash = await bcrypt.hash(password, 10);
   const user = await User.create({
-    name,
-    surname,
-    nationalIdCode: nationalIdCode || nationalId,
-    phoneNumber: phoneNumber || phone,
+    ...profile,
     username,
     passwordHash,
-    email,
-    address,
-    preferredMessenger: normalizePreferredMessengers(preferredMessenger),
     role: userRole._id
   });
 
@@ -75,8 +99,9 @@ const getAllUsers = async (query) => {
   if (query.search) {
     filter.$or = [
       { name: new RegExp(query.search, 'i') },
-      { surname: new RegExp(query.search, 'i') },
-      { username: new RegExp(query.search, 'i') }
+      { username: new RegExp(query.search, 'i') },
+      { nationalIdCode: new RegExp(query.search, 'i') },
+      { phoneNumber: new RegExp(query.search, 'i') }
     ];
   }
   if (query.isActive !== undefined) filter.isActive = query.isActive === 'true';
@@ -92,20 +117,12 @@ const getAllUsers = async (query) => {
 const searchUsers = async (query) => getAllUsers(query);
 
 const createUserAdmin = async (body) => {
+  const profile = pickProfileFields(body);
   const {
-    name,
-    surname,
-    nationalIdCode,
-    nationalId,
-    phoneNumber,
-    phone,
     username: requestedUsername,
     password: requestedPassword,
-    email,
     roleId,
-    role,
-    address,
-    preferredMessenger,
+    role
   } = body;
 
   let roleObj = null;
@@ -120,25 +137,18 @@ const createUserAdmin = async (body) => {
   const plainPassword = (requestedPassword && String(requestedPassword).trim()) || generateSimplePassword();
   const username = await allocateUniqueUsername(requestedUsername);
   const passwordHash = await bcrypt.hash(plainPassword, 10);
-  const resolvedPhone = phoneNumber || phone;
 
   const user = await User.create({
-    name,
-    surname,
-    nationalIdCode: nationalIdCode || nationalId,
-    phoneNumber: resolvedPhone,
+    ...profile,
     username,
     passwordHash,
-    email,
-    address,
-    preferredMessenger: normalizePreferredMessengers(preferredMessenger),
-    role: roleObj._id,
+    role: roleObj._id
   });
 
   try {
-    await sendAccountCreatedSms(resolvedPhone, username, plainPassword);
+    await sendAccountCreatedSms(profile.phoneNumber, username, plainPassword);
   } catch (err) {
-    logger.error(`[createUserAdmin] Account SMS failed for ${resolvedPhone}: ${err.message}`);
+    logger.error(`[createUserAdmin] Account SMS failed for ${profile.phoneNumber}: ${err.message}`);
   }
 
   const created = await User.findById(user._id).select(SAFE_FIELDS).populate(POPULATE_ROLE).lean();
@@ -146,8 +156,8 @@ const createUserAdmin = async (body) => {
     ...created,
     generatedCredentials: {
       username,
-      password: plainPassword,
-    },
+      password: plainPassword
+    }
   };
 };
 
@@ -164,6 +174,9 @@ const updateUser = async (id, body) => {
     passwordHash,
     refreshTokens,
     preferredMessenger,
+    surname,
+    name,
+    gender,
     _id,
     id: bodyId,
     createdAt,
@@ -173,8 +186,12 @@ const updateUser = async (id, body) => {
   } = body;
 
   const update = { ...rest };
+  const mergedName = mergeFullName(name, surname);
+  if (mergedName) update.name = mergedName;
+
+  const normalizedGender = normalizeGender(gender);
+  if (normalizedGender) update.gender = normalizedGender;
 
-  // Map frontend field aliases to schema fields
   if (nationalIdCode || nationalId) {
     update.nationalIdCode = nationalIdCode || nationalId;
   }
@@ -188,12 +205,13 @@ const updateUser = async (id, body) => {
     update.preferredMessenger = normalizePreferredMessengers(preferredMessenger) || [];
   }
 
-  // Never overwrite username with an empty value on update
   if (typeof username === 'string' && username.trim()) {
     update.username = username.trim();
   }
 
-  // Strip empty strings so required validators are not tripped
+  // Legacy field — drop if clients still send it
+  delete update.surname;
+
   Object.keys(update).forEach((key) => {
     if (key === 'preferredMessenger') return;
     if (update[key] === '' || update[key] === null || update[key] === undefined) {
@@ -218,7 +236,7 @@ const deleteUser = async (id) => {
   if (!user) throw new AppError('USER_NOT_FOUND');
 };
 
-const enrollUserInCourse = async (userId, courseId, actorId) => {
+const enrollUserInCourse = async (userId, courseId) => {
   const user = await User.findById(userId);
   if (!user) throw new AppError('USER_NOT_FOUND');
 
@@ -229,4 +247,13 @@ const enrollUserInCourse = async (userId, courseId, actorId) => {
   return User.findById(userId).select(SAFE_FIELDS).populate('courses').lean();
 };
 
-module.exports = { signUp, getUserById, getAllUsers, searchUsers, createUserAdmin, updateUser, deleteUser, enrollUserInCourse };
+module.exports = {
+  signUp,
+  getUserById,
+  getAllUsers,
+  searchUsers,
+  createUserAdmin,
+  updateUser,
+  deleteUser,
+  enrollUserInCourse
+};