feat(programs): undergrad program requirements (#103)

## Description

We scrape and serve undergraduate requirements, i.e. UC and GE
requirements, which are required to earn any undergraduate degree but
are not considered part of any major.

* The DegreeWorks scraper now acquires this data and saves it to the new
`school_requirement` table.
* A new requirement type, the marker requirement, is introduced. This
represents any requirement which is to be marked manually by an advisor
and is not completed by coursework. **This PR conjectures that such
requirements, when not complete, are labelled as "Incomplete".** Based
on my own view of DegreeWorks I only observe "Complete", so this other
variant is a guess.
* We now systematically replace the word "satisfied" with "required",
when preceded by a space, from the name of any requirement. This is
because the names of some requirements, such as American History and
American Institutions, change when they are completed. If not addressed,
this would make scraping results dependent on the actual progress of the
account we scrape from. The word "satisfied" does not currently appear
in any major, minor, or specialization requirement and \*only* in newly
scraped requirements.
* This new marker type is a new member of the requirement type union on
REST and GraphQL, comprising a **breaking change**.
* The new REST endpoint `/v2/rest/programs/ugradRequirements` is
created, taking an `id` which is `"UC" | "GE"`. I argue it is reasonable
to enumerate the options because a violation of this assumption would be
a serious overhaul to the structure of DegreeWorks.
* The GraphQL resource `ugradRequirements` is created with similar
interface.

## Related Issue

Completes #102.

## Motivation and Context

pov: PP asks you for even more features

## How Has This Been Tested?

1. The scraper was rerun to acquire the new data (if you're short on
time, only scrape the new data).
2. Run the new REST and GraphQL.

## Screenshots (if appropriate):

REST:

![Screenshot_20250128_211602](https://github.com/user-attachments/assets/00fa2e95-a436-4626-bc79-d21845a3a8ce)

GraphQL:

![Screenshot_20250128_211541](https://github.com/user-attachments/assets/242ff133-b4f0-4ecb-a9d6-df1fcf1d6cb9)

## Types of changes

- [ ] Bug fix (non-breaking change which fixes an issue)
- [x] New feature (non-breaking change which adds functionality)
- [x] Breaking change (fix or feature that would cause existing
functionality to change)

## Checklist:

- [x] My code involves a change to the database schema.
- [ ] My code requires a change to the documentation.

apps/api/src/graphql/resolvers/programs.ts

@@ -4,10 +4,13 @@ import {
   majorsQuerySchema,
   minorRequirementsQuerySchema,
   minorsQuerySchema,
+  type programRequirementSchema,
   specializationRequirementsQuerySchema,
   specializationsQuerySchema,
+  ugradRequirementsQuerySchema,
 } from "$schema";
 import { ProgramsService } from "$services";
+import type { z } from "@hono/zod-openapi";
 import { GraphQLError } from "graphql/error";
 
 export const programResolvers = {
@@ -72,17 +75,31 @@ export const programResolvers = {
         });
       return res;
     },
+    ugradRequirements: async (_: unknown, args: { query?: unknown }, { db }: GraphQLContext) => {
+      const parsedArgs = ugradRequirementsQuerySchema.parse(args?.query);
+      const service = new ProgramsService(db);
+      const res = await service.getUgradRequirements(parsedArgs);
+      if (!res)
+        throw new GraphQLError("Undergraduate requirements block not found", {
+          extensions: { code: "NOT_FOUND" },
+        });
+      return res;
+    },
   },
   ProgramRequirement: {
     // x outside this typehint is malformed data; meh
-    __resolveType: (x: { requirementType: "Course" | "Unit" | "Group" }) => {
+    __resolveType: (x: {
+      requirementType: z.infer<typeof programRequirementSchema>["requirementType"];
+    }) => {
       switch (x?.requirementType) {
         case "Course":
           return "ProgramCourseRequirement";
         case "Unit":
           return "ProgramUnitRequirement";
         case "Group":
           return "ProgramGroupRequirement";
+        case "Marker":
+          return "ProgramMarkerRequirement";
       }
     },
   },

apps/api/src/graphql/schema/programs.ts

@@ -54,14 +54,28 @@ type ProgramGroupRequirement implements ProgramRequirementBase @cacheControl(max
     requirements: JSON!
 }
 
-union ProgramRequirement = ProgramCourseRequirement | ProgramUnitRequirement | ProgramGroupRequirement
+type ProgramMarkerRequirement implements ProgramRequirementBase @cacheControl(maxAge: 86400) {
+    label: String!
+}
+
+union ProgramRequirement = ProgramCourseRequirement | ProgramUnitRequirement | ProgramGroupRequirement | ProgramMarkerRequirement
 
 type Program @cacheControl(maxAge: 86400) {
     id: String!
     name: String!
     requirements: [ProgramRequirement]!
 }
 
+enum UgradRequirementsBlockId {
+    UC
+    GE
+}
+
+type UgradRequirements @cacheControl(maxAge: 86400) {
+    id: UgradRequirementsBlockId!,
+    requirements: [ProgramRequirement!]!,
+}
+
 input ProgramRequirementsQuery {
     programId: String!
 }
@@ -78,12 +92,17 @@ input SpecializationsQuery {
     majorId: String!
 }
 
+input UgradRequrementsQuery {
+    id: UgradRequirementsBlockId!
+}
+
 extend type Query {
     majors(query: MajorsQuery): [MajorPreview!]!
     minors(query: MinorsQuery): [MinorPreview!]!
     specializations(query: SpecializationsQuery): [SpecializationPreview!]!
     major(query: ProgramRequirementsQuery!): Program!
     minor(query: ProgramRequirementsQuery!): Program!
     specialization(query: ProgramRequirementsQuery!): Program!
+    ugradRequirements(query: UgradRequrementsQuery!): UgradRequirements!
 }
 `;

apps/api/src/rest/routes/programs.ts

@@ -16,6 +16,8 @@ import {
   specializationRequirementsResponseSchema,
   specializationsQuerySchema,
   specializationsResponseSchema,
+  ugradRequirementsQuerySchema,
+  ugradRequirementsResponseSchema,
 } from "$schema";
 import { ProgramsService } from "$services";
 import { OpenAPIHono, createRoute } from "@hono/zod-openapi";
@@ -183,6 +185,36 @@ const specializationRequirements = createRoute({
   },
 });
 
+const ugradRequirements = createRoute({
+  summary: "Retrieve undergraduate requirements",
+  operationId: "ugradRequirements",
+  tags: ["Programs"],
+  method: "get",
+  path: "/ugradRequirements",
+  description: "Retrieve requirements external to, but required for, for all undergraduate degrees",
+  request: { query: ugradRequirementsQuerySchema },
+  responses: {
+    200: {
+      content: {
+        "application/json": { schema: responseSchema(ugradRequirementsResponseSchema) },
+      },
+      description: "Successful operation",
+    },
+    404: {
+      content: { "application/json": { schema: errorSchema } },
+      description: "Specialization not found",
+    },
+    422: {
+      content: { "application/json": { schema: errorSchema } },
+      description: "Parameters failed validation",
+    },
+    500: {
+      content: { "application/json": { schema: errorSchema } },
+      description: "Server error occurred",
+    },
+  },
+});
+
 programsRouter.get(
   "*",
   productionCache({ cacheName: "anteater-api", cacheControl: "max-age=86400" }),
@@ -254,4 +286,19 @@ programsRouter.openapi(specializationRequirements, async (c) => {
       );
 });
 
+programsRouter.openapi(ugradRequirements, async (c) => {
+  const query = c.req.valid("query");
+  const service = new ProgramsService(database(c.env.DB.connectionString));
+  const res = await service.getUgradRequirements(query);
+  return res
+    ? c.json({ ok: true, data: ugradRequirementsResponseSchema.parse(res) }, 200)
+    : c.json(
+        {
+          ok: false,
+          message: "Couldn't find this undergraduate requirements block; check your ID?",
+        },
+        404,
+      );
+});
+
 export { programsRouter };

apps/api/src/schema/programs.ts

@@ -44,6 +44,12 @@ export const specializationRequirementsQuerySchema = z.object({
   }),
 });
 
+export const UgradRequirementsBlockIds = ["UC", "GE"] as const;
+
+export const ugradRequirementsQuerySchema = z.object({
+  id: z.enum(UgradRequirementsBlockIds).openapi({ description: "The requirements block to fetch" }),
+});
+
 export const programRequirementBaseSchema = z.object({
   label: z.string().openapi({
     description: "Human description of this requirement",
@@ -141,11 +147,25 @@ export const programGroupRequirementSchema: z.ZodType<
     },
   });
 
+export const programMarkerRequirementSchema = programRequirementBaseSchema
+  .extend({
+    requirementType: z.literal("Marker"),
+  })
+  .openapi({
+    description:
+      "A rule which must be marked as complete, e.g the fulfillment of GE VIII (foreign language) via high school credit",
+    example: {
+      label: "Entry Level Writing",
+      requirementType: "Marker",
+    },
+  });
+
 // one day someone will figure out z.discriminatedUnion
 export const programRequirementSchema = z.union([
   programCourseRequirementSchema,
   programUnitRequirementSchema,
   programGroupRequirementSchema,
+  programMarkerRequirementSchema,
 ]);
 
 export const majorsResponseSchema = z.array(
@@ -246,3 +266,10 @@ export const specializationRequirementsResponseSchema = programRequirementsRespo
     example: "CS:Specialization in Bioinformatics",
   }),
 });
+
+export const ugradRequirementsResponseSchema = z.object({
+  id: z.string().openapi({ description: "ID of the requirements block fetched" }),
+  requirements: z
+    .array(programRequirementSchema)
+    .openapi({ description: "The requirements in this requirements block" }),
+});

apps/api/src/services/programs.ts

@@ -5,10 +5,12 @@ import type {
   minorsQuerySchema,
   specializationRequirementsQuerySchema,
   specializationsQuerySchema,
+  ugradRequirementsQuerySchema,
 } from "$schema";
 import type { database } from "@packages/db";
 import { eq, sql } from "@packages/db/drizzle";
-import { degree, major, minor, specialization } from "@packages/db/schema";
+import { degree, major, minor, schoolRequirement, specialization } from "@packages/db/schema";
+import { orNull } from "@packages/stdlib";
 import type { z } from "zod";
 
 export class ProgramsService {
@@ -107,4 +109,17 @@ export class ProgramsService {
 
     return got;
   }
+
+  async getUgradRequirements(query: z.infer<typeof ugradRequirementsQuerySchema>) {
+    const [got] = await this.db
+      .select({
+        id: schoolRequirement.id,
+        requirements: schoolRequirement.requirements,
+      })
+      .from(schoolRequirement)
+      .where(eq(schoolRequirement.id, query.id))
+      .limit(1);
+
+    return orNull(got);
+  }
 }

apps/data-pipeline/degreeworks-scraper/src/components/AuditParser.ts

@@ -1,12 +1,12 @@
 import type { Block, Rule } from "$types";
 import type { database } from "@packages/db";
 import { eq } from "@packages/db/drizzle";
-import { course } from "@packages/db/schema";
 import type {
   DegreeWorksProgram,
   DegreeWorksProgramId,
   DegreeWorksRequirement,
 } from "@packages/db/schema";
+import { course } from "@packages/db/schema";
 
 export class AuditParser {
   private static readonly specOrOtherMatcher = /"type":"(?:SPEC|OTHER)","value":"\w+"/g;
@@ -94,6 +94,16 @@ export class AuditParser {
       .limit(1);
   }
 
+  /**
+   * Certain requirements change label depending on whether they've been fulfilled.
+   * This is undesirable for archival so we will quash these.
+   * @param label The label before transformation.
+   * @private
+   */
+  private static suppressLabelPolymorphism(label: string) {
+    return label.replaceAll(/ Satisfied/g, " Required").replaceAll(/ satisfied/g, " required");
+  }
+
   async ruleArrayToRequirements(ruleArray: Rule[]) {
     const ret: DegreeWorksRequirement[] = [];
     for (const rule of ruleArray) {
@@ -129,14 +139,14 @@ export class AuditParser {
             .map(([x]) => x);
           if (rule.requirement.classesBegin) {
             ret.push({
-              label: rule.label,
+              label: AuditParser.suppressLabelPolymorphism(rule.label),
               requirementType: "Course",
               courseCount: Number.parseInt(rule.requirement.classesBegin, 10),
               courses,
             });
           } else if (rule.requirement.creditsBegin) {
             ret.push({
-              label: rule.label,
+              label: AuditParser.suppressLabelPolymorphism(rule.label),
               requirementType: "Unit",
               unitCount: Number.parseInt(rule.requirement.creditsBegin, 10),
               courses,
@@ -146,7 +156,7 @@ export class AuditParser {
         }
         case "Group": {
           ret.push({
-            label: rule.label,
+            label: AuditParser.suppressLabelPolymorphism(rule.label),
             requirementType: "Group",
             requirementCount: Number.parseInt(rule.requirement.numberOfGroups),
             requirements: await this.ruleArrayToRequirements(rule.ruleArray),
@@ -155,20 +165,31 @@ export class AuditParser {
         }
         case "IfStmt": {
           const rules = this.flattenIfStmt([rule]);
-          if (rules.length > 1 && !rules.some((x) => x.ruleType === "Block")) {
-            ret.push({
-              label: "Select 1 of the following",
-              requirementType: "Group",
-              requirementCount: 1,
-              requirements: await this.ruleArrayToRequirements(rules),
-            });
+          if (!rules.some((x) => x.ruleType === "Block")) {
+            if (rules.length > 1) {
+              ret.push({
+                label: "Select 1 of the following",
+                requirementType: "Group",
+                requirementCount: 1,
+                requirements: await this.ruleArrayToRequirements(rules),
+              });
+            } else if (rules.length === 1) {
+              ret.push(...(await this.ruleArrayToRequirements(rules)));
+            }
           }
           break;
         }
+        case "Complete":
+        case "Incomplete":
+          ret.push({
+            label: AuditParser.suppressLabelPolymorphism(rule.label),
+            requirementType: "Marker",
+          });
+          break;
         case "Subset": {
           const requirements = await this.ruleArrayToRequirements(rule.ruleArray);
           ret.push({
-            label: rule.label,
+            label: AuditParser.suppressLabelPolymorphism(rule.label),
             requirementType: "Group",
             requirementCount: Object.keys(requirements).length,
             requirements,

apps/data-pipeline/degreeworks-scraper/src/components/DegreeworksClient.ts

@@ -37,6 +37,44 @@ export class DegreeworksClient {
 
   sleep = (ms: number = this.delay) => new Promise((r) => setTimeout(r, ms));
 
+  static formatQueryParams(params: Record<string, string>) {
+    return Object.entries(params)
+      .map((kv) => kv.map(encodeURIComponent).join("="))
+      .join("&");
+  }
+
+  async getUgradRequirements(): Promise<[Block, Block] | undefined> {
+    const params = DegreeworksClient.formatQueryParams({
+      studentId: this.studentId,
+      // more schools are possible, see this.getMapping("schools"), but we want undergrad requirements
+      school: "U",
+      // there is no difference regardless of which of the four bachelor's degrees we ask for: BA, BFA, BMUS, BS
+      degree: "BS",
+    });
+    const res = await fetch(`${DegreeworksClient.AUDIT_URL}?${params}`, {
+      method: "GET",
+      headers: this.headers,
+    });
+    await this.sleep();
+
+    const json: DWAuditResponse = await res.json().catch(() => ({ error: "" }));
+    if ("error" in json) {
+      return;
+    }
+
+    // "DEGREE" block doesn't contain any material requirements, "SCHOOL" block has what we need
+    const ucRequirements = json.blockArray.find((b) => b.requirementType === "SCHOOL");
+    if (!ucRequirements) {
+      return;
+    }
+    const geRequirements = json.blockArray.find((b) => b.requirementType === "PROGRAM");
+    if (!geRequirements) {
+      return;
+    }
+
+    return [ucRequirements, geRequirements];
+  }
+
   async getMajorAudit(
     degree: string,
     school: string,