fixed issues and extracted another helper function

Author: samuel-lao

frontend/degree-plan/pages/OnboardingPage.tsx

@@ -9,9 +9,10 @@ import { polyfillPromiseWithResolvers } from "./polyfilsResolver";
 
 import "core-js/full/promise/with-resolvers.js";
 
-import { parseItems, parseTranscript } from "../utils/parseUtils";
+import { parseItems, parseTranscript, ParsedText, flattenParsedText } from "../utils/parseUtils";
 import WelcomeLayout from "@/components/OnboardingPanels/WelcomePanel";
 import CreateWithTranscriptPanel from "@/components/OnboardingPanels/CreateWithTranscriptPanel";
+
 polyfillPromiseWithResolvers();
 
 pdfjs.GlobalWorkerOptions.workerSrc = `//unpkg.com/pdfjs-dist@${pdfjs.version}/legacy/build/pdf.worker.min.mjs`;
@@ -44,24 +45,26 @@ const OnboardingPage = ({
   >(`/api/degree/degrees`);
 
   // TRANSCRIPT PARSING
-  const total = useRef<any>({});
+  const total = useRef<Record<number, ParsedText[]>>({});
   const addText = (items: any[], index: number) => {
-    const allText: any = parseItems(items);
-    let textResult = [];
-    for (let col in allText) {
-      let poses = Object.keys(allText[col]).reverse();
-      for (let i in poses) {
-        textResult.push(allText[col][poses[i]].join("").toLowerCase());
-      }
-      total.current[index] = textResult;
-    }
+    const parsed = parseItems(items);
+    total.current[index] = total.current[index] ?? [];
+    total.current[index].push(parsed);
 
     // If all pages have been read, begin to parse text from transcript
     if (Object.keys(total.current).length === numPages) {
-      let all: any = [];
-      for (let key in Object.keys(total.current).sort()) {
-        all = all.concat(total.current[key]);
-      }
+      let all: string[] = [];
+      const sortedPageIndexes = Object.keys(total.current)
+        .map((key) => Number(key))
+        .sort((a, b) => a - b);
+
+      sortedPageIndexes.forEach((pageIndex) => {
+        const pageEntries = total.current[pageIndex];
+        if (!pageEntries) return;
+        pageEntries.forEach((pageText) => {
+          all = all.concat(flattenParsedText(pageText));
+        });
+      });
 
       const {
         scrapedCourses,

frontend/degree-plan/utils/parseUtils.ts

@@ -17,10 +17,35 @@ type DegreeOption = {
   label: string;
 };
 
+type ParsedTextColumn = Record<string, string[]>;
+
+export type ParsedText = {
+  col0: ParsedTextColumn;
+  col1: ParsedTextColumn;
+};
+
+// Given a parsed text object, return a flattened array of strings.
+export const flattenParsedText = (parsedText: ParsedText): string[] => {
+  const columns: (keyof ParsedText)[] = ["col0", "col1"];
+  const flattened: string[] = [];
+
+  columns.forEach((column) => {
+    const columnRows = parsedText[column];
+    const poses = Object.keys(columnRows).reverse();
+    poses.forEach((pose) => {
+      const row = columnRows[pose];
+      flattened.push(row.join("").toLowerCase());
+    });
+  });
+
+  return flattened;
+};
+
+
 // Given a list of line items from the PDF, return an object of columns and the lines in each column.
 export const parseItems = (items: LineItem[]) => {
   // At most the transcript will have two columns - we account for that here.
-  let allText: { col0: string[][]; col1: string[][] } = { col0: [], col1: [] };
+  let allText: ParsedText = { col0: {}, col1: {} };
 
   // Find x value for when second column begins using convenient lines.
   let maxCol = items.reduce(function (acc, el) {
@@ -41,7 +66,7 @@ export const parseItems = (items: LineItem[]) => {
 
     // Ignore potential high school program transcript
     if (items[i].str === "Level:High School") {
-      allText[currentCol] = [];
+      allText[currentCol] = {};
       break;
     }
     if (pos in allText[currentCol])