refactored parseUtils

Author: samuel-lao

frontend/degree-plan/pages/OnboardingPage.tsx

@@ -46,7 +46,7 @@ const OnboardingPage = ({
   // TRANSCRIPT PARSING
   const total = useRef<any>({});
   const addText = (items: any[], index: number) => {
-    const allText: any = parseItems(items, index);
+    const allText: any = parseItems(items);
     let textResult = [];
     for (let col in allText) {
       let poses = Object.keys(allText[col]).reverse();

frontend/degree-plan/utils/parseUtils.ts

@@ -2,9 +2,25 @@ import { createMajorLabel } from "@/components/FourYearPlan/DegreeModal";
 import { DegreeListing, SchoolOption } from "@/types";
 const { closest } = require("fastest-levenshtein");
 
-export const parseItems = (items: any[], index: number) => {
+type LineItem = {
+  dir: string;
+  fontName: string;
+  hasEOL: boolean;
+  height: number;
+  str: string;
+  transform: number[];
+  width: number;
+};
+
+type DegreeOption = {
+  value: DegreeListing;
+  label: string;
+};
+
+// Given a list of line items from the PDF, return an object of columns and the lines in each column.
+export const parseItems = (items: LineItem[]) => {
   // At most the transcript will have two columns - we account for that here.
-  let allText: any = { col0: [], col1: [] };
+  let allText: { col0: string[][]; col1: string[][] } = { col0: [], col1: [] };
 
   // Find x value for when second column begins using convenient lines.
   let maxCol = items.reduce(function (acc, el) {
@@ -21,7 +37,7 @@ export const parseItems = (items: any[], index: number) => {
     let col = items[i]?.transform[4];
     let pos = items[i]?.transform[5];
 
-    let currentCol = col < maxCol ? "col0" : "col1";
+    let currentCol: "col0" | "col1" = col < maxCol ? "col0" : "col1";
 
     // Ignore potential high school program transcript
     if (items[i].str === "Level:High School") {
@@ -33,14 +49,17 @@ export const parseItems = (items: any[], index: number) => {
     else allText[currentCol][pos] = [items[i]?.str];
   }
 
+  console.log(allText);
   return allText;
 };
 
+// Given a list of degrees, a list of schools, and a starting year,
+// return a list of relevant possible majors.
 export const getMajorOptions = (
   degrees: DegreeListing[] | undefined,
   schools: SchoolOption[],
-  startingYear: any
-) => {
+  startingYear: number
+): DegreeOption[] | undefined => {
   const majorOptions = degrees
     ?.filter((d) => schools.map((s) => s.value).includes(d.degree))
     .sort((d) => Math.abs((startingYear ? startingYear : d.year) - d.year))
@@ -52,138 +71,91 @@ export const getMajorOptions = (
   return majorOptions;
 };
 
-export const parseTranscript = (
-  textResult: any,
-  degrees: DegreeListing[] | undefined
-) => {
-  let separatedCourses: any = [];
-  let startYear: number = 0;
-  let tempSchools: any = [];
-  let detectedMajors: string[] = [];
-  let detectedConcentrations: string[] = [];
-
-  for (let l in textResult) {
-    // SCRAPE SCHOOL
-    if (textResult[l].replaceAll(" ", "").includes("program:")) {
-      let program = textResult[l].replace(/^.*?:\s*/, "");
-      if (program.includes("arts"))
-        tempSchools.push({ value: "BA", label: "Arts & Sciences" });
-
-      if (program.includes("school of engineering and applied science")) {
-        if (
-          textResult[parseInt(l) + 1].includes(
-            "bachelor of science in engineering"
-          )
-        )
-          tempSchools.push({ value: "BSE", label: "Engineering BSE" });
-        else tempSchools.push({ value: "BAS", label: "Engineering BAS" });
-      }
-      if (program.includes("wharton"))
-        tempSchools.push({ value: "BS", label: "Wharton" });
-      if (program.includes("nursing"))
-        tempSchools.push({ value: "BSN", label: "Nursing" });
-    }
+// Given a string[] where we're guaranteed to have a school line, return a list of scraped schools.
+const checkSchool = (textResult: string[], l: number) => {
+  const tempSchools = [];
+  let program = textResult[l].replace(/^.*?:\s*/, "");
+  if (program.includes("arts"))
+    tempSchools.push({ value: "BA", label: "Arts & Sciences" });
+  if (program.includes("school of engineering and applied science")) {
+    if (textResult[l + 1].includes("bachelor of science in engineering"))
+      tempSchools.push({ value: "BSE", label: "Engineering BSE" });
+    else tempSchools.push({ value: "BAS", label: "Engineering BAS" });
+  }
+  if (program.includes("wharton"))
+    tempSchools.push({ value: "BS", label: "Wharton" });
+  if (program.includes("nursing"))
+    tempSchools.push({ value: "BSN", label: "Nursing" });
 
-    // SCRAPE MAJOR
-    if (textResult[l].includes("major")) {
-      detectedMajors.push(textResult[l].replace(/^.*?:\s*/, ""));
-    }
+  return tempSchools;
+};
 
-    // SCRAPE CONCENTRATION
-    if (textResult[l].includes("concentration")) {
-      detectedConcentrations.push(textResult[l].replace(/^.*?:\s*/, ""));
+// Given a string[] where we're guaranteed to have a transfer credit line,
+// return a list of scraped AP and transfer courses. Stops when we reach potentially
+// non-transfer credit lines.
+const getAPAndTransferCourses = (textResult: any, l: number) => {
+  let courses: { [key: string]: string } = {};
+  let truncatedTranscript = textResult.slice(l + 1);
+  for (let line of truncatedTranscript) {
+    // Match lines following course code format
+    let courseMatch = line.match(/\b\w+\s\d{3,4}\b/);
+    if (
+      courseMatch &&
+      // Match lines following [term] [year] format
+      !/(fall|spring|summer)\s\d{4}/i.test(courseMatch)
+    ) {
+      courses[courseMatch[0]] = "_TRAN";
+    } else if (line.includes("institution credit")) {
+      break;
     }
+  }
+  return courses;
+};
 
-    // SCRAPE AP AND TRANSFER CREDIT
-    if (textResult[l].includes("transfer credit")) {
-      let truncatedTranscript = textResult.slice(parseInt(l) + 1);
-      let courses = [];
-      for (let line of truncatedTranscript) {
-        // Match lines following course code format
-        let courseMatch = line.match(/\b\w+\s\d{3,4}\b/);
-        if (
-          courseMatch &&
-          // Match lines following [term] [year] format
-          !/(fall|spring|summer)\s\d{4}/i.test(courseMatch)
-        ) {
-          courses.push(courseMatch[0]);
-        } else if (line.includes("institution credit")) {
-          separatedCourses["_TRAN"] = courses;
-          break;
-        }
+// Given a string[] where what follows is guaranteed to be the student's non-transfer courses,
+// return an array of semester + courses objects.
+const getCourseToSem = (truncatedTranscript: string[]) => {
+  let firstNonSummerSemReached = false;
+  let currentSem = "";
+  let courseToSem: { [key: string]: string } = {};
+  for (let line of truncatedTranscript) {
+    if (/(fall|spring|summer)\s\d{4}/i.test(line)) {
+      currentSem = line;
+      if (!firstNonSummerSemReached && !currentSem.includes("summer")) {
+        firstNonSummerSemReached = true;
       }
-    }
-
-    // SCRAPE COURSES (BY SEM)
-    let firstNonSummerSemReached = false;
-    
-    let courseToSem: { [key: string]: string } = {};
-    
-    if (textResult[l].includes("institution credit")) {
-      let truncatedTranscript = textResult.slice(parseInt(l) + 1);
-      let currentSem = "";
-      for (let line of truncatedTranscript) {
-        if (/(fall|spring|summer)\s\d{4}/i.test(line)) {
-          currentSem = line;
-          if (!firstNonSummerSemReached && !currentSem.includes("summer")) {
-            firstNonSummerSemReached = true;
-          }
-          // Only start creating sems after first non-summer semester is reached
-          if (firstNonSummerSemReached) {
-            separatedCourses[currentSem] = [];
-          }
-        } else {
-          let courseMatch = line.match(/\b\w+\s\d{3,4}\b/);
-          
-          if (courseMatch) {
-            // Check if course didn't get an F or a W. If so, add to current sem or _TRAN           
-            if (!(line[line.length - 1] == "f" || line[line.length - 1] == "w")) {
-              // TODO: We don't yet have a way to track courses that can be taken multiple times, 
-              // so we store a course that appears multiple times only in the most recent semester it appears in.
-              if (courseMatch[0] in courseToSem) {
-                const prevSem = courseToSem[courseMatch[0]];
-                separatedCourses[currentSem].push(courseMatch[0]);
-                courseToSem[courseMatch[0]] = currentSem;
-                separatedCourses[prevSem] = separatedCourses[prevSem].filter((c: string) => c !== courseMatch[0]);
-              } else {
-                // Add all pre-college courses to _TRAN semester
-                if (firstNonSummerSemReached) {
-                  separatedCourses[currentSem].push(courseMatch[0]);
-                  courseToSem[courseMatch[0]] = currentSem;
-                } else {
-                  separatedCourses["_TRAN"].push(courseMatch[0]);
-                  courseToSem[courseMatch[0]] = "_TRAN";
-                }
-              }
+    } else {
+      let courseMatch = line.match(/\b[A-Za-z]{2,}\s\d{3,4}\b/);
+      if (courseMatch) {
+        // Check if course didn't get an F or a W. If so, add to current sem or _TRAN
+        if (!(line[line.length - 1] == "f" || line[line.length - 1] == "w")) {
+          // TODO: We don't yet have a way to track courses that can be taken multiple times,
+          // so we store a course that appears multiple times only in the most recent semester it appears in.
+          if (courseMatch[0] in courseToSem) {
+            courseToSem[courseMatch[0]] = currentSem;
+          } else {
+            // Add all pre-college courses to _TRAN semester
+            if (firstNonSummerSemReached) {
+              courseToSem[courseMatch[0]] = currentSem;
+            } else {
+              courseToSem[courseMatch[0]] = "_TRAN";
             }
           }
         }
       }
-
-      // Remove any empty semesters (handles edge case where user fails/withdraws from all courses in a semester)
-      for (let sem of Object.keys(separatedCourses)) {
-        if (separatedCourses[sem].length == 0) {
-          delete separatedCourses[sem];
-        }
-      }
-      
-      separatedCourses = Object.keys(separatedCourses).map(
-        (key) => ({ sem: key, courses: separatedCourses[key] })
-      );
-
-      // SCRAPE START YEAR AND INFER GRAD YEAR
-      let years = separatedCourses.map((e: any, i: number) => {
-        return parseInt(e.sem.replace(/\D/g, ""));
-      });
-      years.shift();
-      startYear = Math.min(...years);
     }
   }
+  return courseToSem;
+};
 
-  // Match majors
-  let detectedMajorsOptions = [];
-
-  let possibleDegrees = getMajorOptions(degrees, tempSchools, startYear);
+// Given a list of detected majors, a list of detected concentrations, and a list of possible degrees,
+// return a list of detected majors options.
+const detectMajors = (
+  detectedMajors: string[],
+  detectedConcentrations: string[],
+  possibleDegrees: DegreeOption[] | undefined
+) => {
+  const detectedMajorsOptions = [];
   for (let i in detectedMajors) {
     let m =
       detectedMajors[i] +
@@ -206,8 +178,85 @@ export const parseTranscript = (
       if (majorOption) detectedMajorsOptions.push(majorOption);
     }
   }
+
+  return detectedMajorsOptions;
+};
+
+// Given a list of lines from the PDF and a list of possible degrees,
+// return a scraped information.
+export const parseTranscript = (
+  textResult: string[],
+  degrees: DegreeListing[] | undefined
+) => {
+  let courseToSem: { [key: string]: string } = {};
+  let startYear: number = 0;
+  let tempSchools: { value: string; label: string }[] = [];
+  let detectedMajors: string[] = [];
+  let detectedConcentrations: string[] = [];
+
+  for (let l = 0; l < textResult.length; l++) {
+    if (textResult[l].replaceAll(" ", "").includes("program:")) {
+      tempSchools = tempSchools.concat(checkSchool(textResult, l));
+    }
+
+    if (textResult[l].includes("major")) {
+      detectedMajors.push(textResult[l].replace(/^.*?:\s*/, ""));
+    }
+
+    if (textResult[l].includes("concentration")) {
+      detectedConcentrations.push(textResult[l].replace(/^.*?:\s*/, ""));
+    }
+
+    if (textResult[l].includes("transfer credit")) {
+      courseToSem = {
+        ...courseToSem,
+        ...getAPAndTransferCourses(textResult, l),
+      };
+    }
+
+    if (textResult[l].includes("institution credit")) {
+      courseToSem = {
+        ...courseToSem,
+        ...getCourseToSem(textResult.slice(l + 1)),
+      };
+    }
+  }
+
+  const separatedCourses = Object.entries(courseToSem).reduce(
+    (acc, [course, sem]) => {
+      const trimmedSem = sem.trim();
+      if (!acc[trimmedSem]) acc[trimmedSem] = [];
+      acc[trimmedSem].push(course);
+      return acc;
+    },
+    {} as { [key: string]: string[] }
+  );
+
+  const formattedSeparatedCourses = Object.entries(separatedCourses).map(
+    ([sem, courses]) => ({
+      sem,
+      courses,
+    })
+  );
+
+  // Scrape start year and infer grad year
+  let years = formattedSeparatedCourses.map(
+    (e: { sem: string; courses: string[] }, i: number) => {
+      return parseInt(e.sem.replace(/\D/g, ""));
+    }
+  );
+  years.shift();
+  startYear = Math.min(...years);
+
+  let possibleDegrees = getMajorOptions(degrees, tempSchools, startYear);
+  let detectedMajorsOptions = detectMajors(
+    detectedMajors,
+    detectedConcentrations,
+    possibleDegrees
+  );
+
   return {
-    scrapedCourses: separatedCourses,
+    scrapedCourses: formattedSeparatedCourses,
     startYear: startYear,
     scrapedSchools: tempSchools,
     detectedMajorsOptions: detectedMajorsOptions,