Merge pull request #209 from ufabc-next/components-auto-sync

feat: sync automatically

Author: Joabesv

apps/core/src/jobs/components-create.ts

@@ -1,13 +1,50 @@
+import type { Types } from 'mongoose';
+
 import { defineJob } from '@next/queues/client';
 
 import { UfabcParserConnector } from '@/connectors/ufabc-parser.js';
 import { JOB_NAMES } from '@/constants.js';
 import { ComponentModel, type Component } from '@/models/Component.js';
+import { TeacherModel } from '@/models/Teacher.js';
 
 import { findOrCreateSubject } from './utils/subject-resolution.js';
 
 const connector = new UfabcParserConnector();
 
+const teacherCache = new Map<string, Types.ObjectId | null>();
+
+async function findTeacher(
+  name: string | null
+): Promise<Types.ObjectId | null> {
+  if (!name) return null;
+
+  const normalizedName = name
+    .toLowerCase()
+    .normalize('NFD')
+    .replace(/[\u0300-\u036f]/g, '');
+
+  if (teacherCache.has(normalizedName)) {
+    return teacherCache.get(normalizedName)!;
+  }
+
+  const teacher = await TeacherModel.findByFuzzName(normalizedName);
+
+  if (!teacher && normalizedName !== '0') {
+    teacherCache.set(normalizedName, null);
+    return null;
+  }
+
+  if (teacher && !teacher.alias.includes(normalizedName)) {
+    await TeacherModel.findByIdAndUpdate(teacher._id, {
+      $addToSet: { alias: [normalizedName, name.toLowerCase()] },
+    });
+  }
+
+  const teacherId = teacher?._id ?? null;
+  teacherCache.set(normalizedName, teacherId);
+  return teacherId;
+}
+
 export const createComponentJob = defineJob(JOB_NAMES.CREATE_COMPONENT).handler(
   async ({ job }) => {
     const { componentId } = job.data;
@@ -24,6 +61,18 @@ export const createComponentJob = defineJob(JOB_NAMES.CREATE_COMPONENT).handler(
       subjectCode
     );
 
+    const professorTeacher = component.teachers?.find(
+      (t) => t.role === 'professor' && !t.isSecondary
+    );
+    const practiceTeacher = component.teachers?.find(
+      (t) => t.role === 'practice' && !t.isSecondary
+    );
+
+    const [teoria, pratica] = await Promise.all([
+      findTeacher(professorTeacher?.name ?? null),
+      findTeacher(practiceTeacher?.name ?? null),
+    ]);
+
     const dbComponent = {
       after_kick: [],
       before_kick: [],
@@ -33,7 +82,10 @@ export const createComponentJob = defineJob(JOB_NAMES.CREATE_COMPONENT).handler(
       turno: component.shift === 'morning' ? 'diurno' : 'noturno',
       turma: component.componentClass,
       vagas: component.vacancies,
-      obrigatorias: component.courses?.filter((c) => c.category === 'mandatory').map((c) => c.UFCourseId) ?? [],
+      obrigatorias:
+        component.courses
+          ?.filter((c) => c.category === 'mandatory')
+          .map((c) => c.UFCourseId) ?? [],
       uf_cod_turma: component.ufClassroomCode,
       campus: component.campus,
       codigo: component.ufComponentCode,
@@ -48,6 +100,8 @@ export const createComponentJob = defineJob(JOB_NAMES.CREATE_COMPONENT).handler(
       year: Number(component.season.split(':')[0]),
       quad: Number(component.season.split(':')[1]),
       season: component.season,
+      teoria,
+      pratica,
     } satisfies Omit<Component, 'createdAt' | 'updatedAt'>;
 
     const createdComponent = await ComponentModel.findOneAndUpdate(

apps/core/src/routes/entities/components/index.ts

@@ -258,7 +258,10 @@ const plugin: FastifyPluginAsyncZodOpenApi = async (app) => {
       );
 
       if (result.matchedCount === 0) {
-        app.log.info({ disciplinaId, season: seasonToUse }, 'No matching component found');
+        app.log.info(
+          { disciplinaId, season: seasonToUse },
+          'No matching component found'
+        );
         return reply.status(404).send({
           error: 'No matching component found',
           disciplinaId,
@@ -267,7 +270,10 @@ const plugin: FastifyPluginAsyncZodOpenApi = async (app) => {
       }
 
       if (result.modifiedCount === 0) {
-        app.log.info({ disciplinaId, season: seasonToUse }, 'Component found but not modified (same groupURL)');
+        app.log.info(
+          { disciplinaId, season: seasonToUse },
+          'Component found but not modified (same groupURL)'
+        );
         return reply.send({
           message: 'Component found but groupURL was already set to this value',
           disciplinaId,
@@ -277,7 +283,11 @@ const plugin: FastifyPluginAsyncZodOpenApi = async (app) => {
       }
 
       app.log.info(
-        { disciplinaId, season: seasonToUse, modifiedCount: result.modifiedCount },
+        {
+          disciplinaId,
+          season: seasonToUse,
+          modifiedCount: result.modifiedCount,
+        },
         'GroupURL updated successfully'
       );
 

apps/core/src/routes/sync/index.ts

@@ -4,9 +4,6 @@ import { createHash } from 'node:crypto';
 
 import { UfabcParserConnector } from '@/connectors/ufabc-parser.js';
 import { ComponentModel, type Component } from '@/models/Component.js';
-import { SubjectModel } from '@/models/Subject.js';
-import { TeacherModel } from '@/models/Teacher.js';
-import { syncComponentsSchema } from '@/schemas/sync/components.js';
 import { syncEnrolledSchema } from '@/schemas/sync/enrolled.js';
 import { syncEnrollmentsSchema } from '@/schemas/sync/enrollments.js';
 
@@ -170,227 +167,6 @@ const plugin: FastifyPluginAsyncZodOpenApi = async (app) => {
     }
   );
 
-  app.put(
-    '/components',
-    {
-      schema: syncComponentsSchema,
-      preHandler: (request, reply) => request.isAdmin(reply),
-    },
-    async (request, reply) => {
-      const { season, hash, ignoreErrors } = request.body;
-      const componentsWithTeachers = await connector.getComponentsV2(season);
-
-      const teacherCache = new Map();
-      const errors: Array<SyncError> = [];
-
-      const findTeacher = async (name: string | null) => {
-        if (!name) return null;
-        const normalizedName = name
-          .toLowerCase()
-          .normalize('NFD')
-          .replace(/\u0300-\u036f/g, '');
-        if (teacherCache.has(normalizedName))
-          return teacherCache.get(normalizedName);
-        // @ts-ignore Complex Type Mismatch
-        const teacher = await TeacherModel.findByFuzzName(normalizedName);
-        if (!teacher && normalizedName !== '0') {
-          app.log.warn({
-            originalName: name,
-            normalizedName,
-          }, 'Teacher not found');
-          errors.push({
-            original: name,
-            parserError: ['Teacher not found in database'],
-            metadata: { normalizedName },
-            type: 'TEACHER_NOT_FOUND',
-          });
-          teacherCache.set(normalizedName, null);
-          return null;
-        }
-        if (teacher && !teacher.alias.includes(normalizedName)) {
-          await TeacherModel.findByIdAndUpdate(teacher._id, {
-            $addToSet: { alias: [normalizedName, name.toLowerCase()] },
-          });
-        }
-        teacherCache.set(normalizedName, teacher?._id ?? null);
-        return teacher?._id ?? null;
-      };
-
-      const components = await Promise.all(
-        componentsWithTeachers.map(async (c) => {
-          const professorTeacher = c.teachers.find(
-            (t) => t.role === 'professor' && !t.isSecondary
-          );
-          const practiceTeacher = c.teachers.find(
-            (t) => t.role === 'practice' && !t.isSecondary
-          );
-
-          const [teoria, pratica] = await Promise.all([
-            findTeacher(professorTeacher?.name ?? null),
-            findTeacher(practiceTeacher?.name ?? null),
-          ]);
-          const dbComponent = await ComponentModel.findOne({
-            season,
-            $or: [
-              { uf_cod_turma: c.ufClassroomCode.toUpperCase() },
-              { uf_cod_turma: c.ufClassroomCode },
-            ],
-          }).lean();
-          let subjectId = null;
-          if (!dbComponent) {
-            app.log.warn({
-              msg: 'Component not found in database',
-              uf_cod_turma: c.ufClassroomCode,
-            });
-            errors.push({
-              original: c.ufClassroomCode,
-              parserError: ['Component not found in database'],
-              type: 'MATCHING_FAILED',
-            });
-            // Normalize the subject code (strip year, uppercase, etc.)
-            const codeMatch = c.ufComponentCode.match(/^(.*?)-\d{2}$/);
-            const normalizedCode = codeMatch ? codeMatch[1] : c.ufComponentCode;
-            const subject = await SubjectModel.findOne({
-              uf_subject_code: { $in: [normalizedCode] },
-            }).lean();
-            if (!subject) {
-              app.log.warn({
-                msg: 'Subject not found for unmatched component',
-                component: c,
-              });
-              errors.push({
-                original: c.name,
-                parserError: ['Subject not found for unmatched component'],
-                type: 'MATCHING_FAILED',
-              });
-            }
-            subjectId = subject?._id;
-            return {
-              disciplina_id: null,
-              campus: c.campus,
-              disciplina: c.name,
-              turno: c.shift,
-              turma: c.componentClass,
-              uf_cod_turma: c.ufClassroomCode,
-              year: Number(season.split(':')[0]),
-              quad: Number(season.split(':')[1]),
-              codigo: c.ufComponentCode,
-              season,
-              teoria,
-              pratica,
-              subject: subjectId,
-              after_kick: [],
-              before_kick: [],
-              alunos_matriculados: [],
-              obrigatorias: [],
-              ideal_quad: false,
-              vagas: c.vacancies,
-              kind: 'file' as 'file' | 'api',
-              flag: 'upsert',
-              ignoreErrors,
-            };
-          }
-          return {
-            ...dbComponent,
-            campus: c.campus,
-            disciplina: c.name,
-            turno: c.shift,
-            turma: c.componentClass,
-            uf_cod_turma: c.ufClassroomCode,
-            teoria,
-            pratica,
-            ignoreErrors,
-          };
-        })
-      );
-
-      if (!ignoreErrors && errors.length > 0) {
-        const teacherErrors = errors.filter(
-          (e) => e.type === 'TEACHER_NOT_FOUND'
-        );
-        const matchingErrors = errors.filter(
-          (e) => e.type === 'MATCHING_FAILED'
-        );
-        return reply.status(403).send({
-          msg: 'Errors found while verifying components',
-          errors: {
-            missingTeachers: [...new Set(teacherErrors.map((e) => e.original))],
-            unmatchedComponents: matchingErrors,
-          },
-          totalErrors: errors.length,
-          breakdown: {
-            teacherErrors: teacherErrors.length,
-            matchingErrors: matchingErrors.length,
-          },
-        });
-      }
-
-      const componentHash = createHash('md5')
-        .update(JSON.stringify(components))
-        .digest('hex');
-      if (componentHash !== hash) {
-        return {
-          hash: componentHash,
-          errors,
-          total: components.length,
-          payload: components,
-        };
-      }
-
-      // Dispatch all jobs, always passing flag and ignoreErrors
-      const dispatchPromises = components.map(async (componentData) => {
-        try {
-          if (!componentData.subject) {
-            request.log.warn({
-              error: 'Component data is missing',
-              component: componentData.disciplina,
-              msg: 'Component data is missing',
-            });
-            throw new Error('Component data is missing');
-          }
-
-          // fuck this
-          // @ts-expect-error
-          await app.job.dispatch('ComponentsTeachersSync', componentData);
-          return { success: true, component: componentData.disciplina };
-        } catch (error) {
-          const errorMsg = `Failed to dispatch component: ${componentData.disciplina}`;
-          if (ignoreErrors) {
-            request.log.warn({
-              error: error instanceof Error ? error.message : String(error),
-              component: componentData.disciplina,
-              msg: `${errorMsg} (ignored)`,
-            });
-            return {
-              success: false,
-              component: componentData.disciplina,
-              ignored: true,
-            };
-          }
-          request.log.error({
-            error: error instanceof Error ? error.message : String(error),
-            component: componentData.disciplina,
-            msg: errorMsg,
-          });
-          throw error;
-        }
-      });
-      const dispatchResults = await Promise.all(dispatchPromises);
-      const successfulDispatches = dispatchResults.filter(
-        (r) => r.success
-      ).length;
-      const ignoredErrors = dispatchResults.filter(
-        (r) => !r.success && r.ignored
-      ).length;
-      return reply.send({
-        dispatched: true,
-        msg: 'Component teacher sync jobs dispatched',
-        totalComponents: components.length,
-        successfulDispatches,
-        ignoredErrors: ignoreErrors ? ignoredErrors : 0,
-      });
-    }
-  );
 
   app.put(
     '/enrolled',