From 418332402b7dd11aa77f3b21d0d768b9eb1a2c6f Mon Sep 17 00:00:00 2001 From: ibrahimjaved12 <109785089+ibrahimjaved12@users.noreply.github.com> Date: Mon, 8 Jan 2024 20:23:07 +0500 Subject: [PATCH] Create OER Commons CSV from OCW (#1) * Initial setup --------- Co-authored-by: Ibrahim Javed --- .gitignore | 7 + .pre-commit-config.yaml | 26 + README.md | 99 +- ocw_oer_export/__init__.py | 9 + ocw_oer_export/cli.py | 38 + ocw_oer_export/client.py | 52 + ocw_oer_export/constants.py | 4 + ocw_oer_export/create_csv.py | 195 +++ ocw_oer_export/create_json.py | 22 + ocw_oer_export/data_handler.py | 35 + .../ocw_topic_to_oer_subject.csv | 329 +++++ ocw_oer_export/utilities/__init__.py | 4 + ocw_oer_export/utilities/delete_file.py | 10 + ocw_oer_export/utilities/text_cleanup.py | 64 + pyproject.toml | 3 + requirements.txt | 21 + setup.cfg | 32 + setup.py | 3 + tests/__init__.py | 0 tests/expected_courses.csv | 25 + tests/sample_courses.json | 1147 +++++++++++++++++ tests/test_api_response.py | 30 + tests/test_csv_creation_from_json.py | 38 + tests/test_description_cleanup.py | 44 + 24 files changed, 2236 insertions(+), 1 deletion(-) create mode 100644 .pre-commit-config.yaml create mode 100644 ocw_oer_export/__init__.py create mode 100644 ocw_oer_export/cli.py create mode 100644 ocw_oer_export/client.py create mode 100644 ocw_oer_export/constants.py create mode 100644 ocw_oer_export/create_csv.py create mode 100644 ocw_oer_export/create_json.py create mode 100644 ocw_oer_export/data_handler.py create mode 100644 ocw_oer_export/mapping_files/ocw_topic_to_oer_subject.csv create mode 100644 ocw_oer_export/utilities/__init__.py create mode 100644 ocw_oer_export/utilities/delete_file.py create mode 100644 ocw_oer_export/utilities/text_cleanup.py create mode 100644 pyproject.toml create mode 100644 requirements.txt create mode 100644 setup.cfg create mode 100644 setup.py create mode 100644 tests/__init__.py create mode 100644 tests/expected_courses.csv create mode 100644 tests/sample_courses.json create mode 100644 tests/test_api_response.py create mode 100644 tests/test_csv_creation_from_json.py create mode 100644 tests/test_description_cleanup.py diff --git a/.gitignore b/.gitignore index 68bc17f..1bcef8b 100644 --- a/.gitignore +++ b/.gitignore @@ -158,3 +158,10 @@ cython_debug/ # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ + +# DS_Store +.DS_Store + +# OER Export JSON and CSV files created +ocw_api_data.json +ocw_oer_export.csv diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..ceef286 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,26 @@ +--- +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.5.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-ast + - id: check-docstring-first + - id: check-merge-conflict + - id: check-yaml + - id: check-added-large-files + - id: debug-statements + - repo: https://github.com/adrienverge/yamllint.git + rev: v1.33.0 + hooks: + - id: yamllint + args: [--format, parsable, -d, relaxed] + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: "v0.1.9" + hooks: + - id: ruff + args: [ --fix ] + - id: ruff-format diff --git a/README.md b/README.md index d5b0208..622d556 100644 --- a/README.md +++ b/README.md @@ -1 +1,98 @@ -# ocw-oer-export \ No newline at end of file +# OCW OER Export + +This demonstration project showcases how to utilize the MIT Open API. It specifically focuses on extracting MIT OpenCourseWare courses' metadata and creating a CSV file for export to OER Commons, aligning with their specific [requirements](https://help.oercommons.org/support/solutions/articles/42000046853-import-resources-with-the-bulk-import-template). + +**SECTIONS** + +1. [Initial Setup & Usage](#initial-setup) +1. [Requirements](#requirements) +1. [Tests](#tests) +1. [Committing & Formatting](#committing-&-formatting) + + +## Initial Setup & Usage + +The _ocw_oer_export_ package is available [on PyPI](link). To install: + +``` +pip install ocw_oer_export +``` + +### Usage as a Python Package + +To use `ocw_oer_export` in your Python code: + +``` +from ocw_oer_export import create_csv +create_csv() +``` + +By default, the `create_csv` function uses `source="api"` and `output_file="ocw_oer_export.csv"`. The `source` parameter can be altered to `source="json"` if a local JSON file of courses' metadata is available. To generate the JSON file: + +``` +from ocw_oer_export import create_json +create_json() +``` + +Then, create the CSV from the JSON file: + +``` +create_csv(source="json") +``` + +### CLI Usage + +`ocw_oer_export` also provides a Command Line Interface (CLI). After installation, you can use the following commands: + +To create the CSV file: + +``` +ocw-oer-export --create_csv +``` + +To generate a JSON file: + +``` +ocw-oer-export --create_json +``` + +To create a CSV file from the local JSON file: + +``` +ocw-oer-export --create_csv --source=json +``` + +## File Output Directory + +When using either the Python package or the CLI, the output files (CSV or JSON) are saved in the current working directory from which it is executed. + +## Requirements + +For successful execution and correct output, ensure the [MIT Open's API](https://mit-open-rc.odl.mit.edu//api/v1/courses/?platform=ocw) contains the following fields: + +`title`, `url`, `description`, `topics`, `course_feature`, `runs: instructors` + +Additionally, the `mapping_files` should be up-to-date. If new topics are added in OCW without corresponding mappings in `ocw_oer_export/mapping_files/ocw_topic_to_oer_subject.csv`, this will lead to `null` entries for those topics in the CSV (`CR_SUBJECT`). + +## Tests + +To run unit tests: + +``` +python -m unittest discover +``` + +## Committing & Formatting + +To ensure commits to GitHub are safe, first install [pre-commit](https://pre-commit.com/): + +``` +pip install pre-commit +pre-commit install +``` + +Running pre-commit can confirm your commit is safe to be pushed to GitHub and correctly formatted: + +``` +pre-commit run --all-files +``` diff --git a/ocw_oer_export/__init__.py b/ocw_oer_export/__init__.py new file mode 100644 index 0000000..c273e19 --- /dev/null +++ b/ocw_oer_export/__init__.py @@ -0,0 +1,9 @@ +__all__ = ["create_json", "create_csv", "main"] + +import logging + +from .create_csv import create_csv +from .create_json import create_json +from .cli import main + +logging.root.setLevel(logging.INFO) diff --git a/ocw_oer_export/cli.py b/ocw_oer_export/cli.py new file mode 100644 index 0000000..61fe627 --- /dev/null +++ b/ocw_oer_export/cli.py @@ -0,0 +1,38 @@ +""" +Command-line interface (CLI) for the OCW OER Export Project. + +This module provides a CLI to generate JSON or CSV files containing +MIT OpenCourseWare courses' metadata. +""" +import argparse +from .create_csv import create_csv +from .create_json import create_json + + +def main(): + """ + Parses command-line arguments and executes the appropriate function. + """ + parser = argparse.ArgumentParser(description="OCW OER Export") + + parser.add_argument("--create_csv", action="store_true", help="Create CSV file") + parser.add_argument("--create_json", action="store_true", help="Create JSON file") + parser.add_argument( + "--source", + choices=["api", "json"], + default="api", + help="Specify data source for CSV creation (default: api)", + ) + + args = parser.parse_args() + + if args.create_csv: + create_csv(source=args.source) + elif args.create_json: + create_json() + else: + parser.print_help() + + +if __name__ == "__main__": + main() diff --git a/ocw_oer_export/client.py b/ocw_oer_export/client.py new file mode 100644 index 0000000..2b44f42 --- /dev/null +++ b/ocw_oer_export/client.py @@ -0,0 +1,52 @@ +""" +Module for interacting with the MIT OpenCourseWare API. +""" +import math +import logging +import requests +from retry import retry +from tqdm import tqdm + +logging.basicConfig(level=logging.ERROR) +logger = logging.getLogger(__name__) + + +@retry(tries=3, delay=2, logger=logger) +def make_request(next_page, page_size): + """ + Make a request to the API with retry logic. + """ + return requests.get(next_page, params={"limit": page_size}, timeout=60) + + +def paginated_response(api_url, page_size=100): + """ + Generate paginated responses from the API. + """ + next_page = api_url + while next_page: + response = make_request(next_page, page_size) + data = response.json() + next_page = data.get("next") + yield data + + +def extract_data_from_api(api_url): + """Extract all data from the MIT OpenCourseWare API.""" + page_size = 100 + pages = paginated_response(api_url, page_size) + + first_page = next(pages) + api_data = first_page.get("results", []) + total_pages = math.ceil(first_page["count"] / page_size) + + # Remaining pages + for page in tqdm( + pages, + desc="Loading data from MIT OCW API", + total=total_pages - 1, + ): + page_results = page.get("results", []) + api_data.extend(page_results) + + return api_data diff --git a/ocw_oer_export/constants.py b/ocw_oer_export/constants.py new file mode 100644 index 0000000..78b08a3 --- /dev/null +++ b/ocw_oer_export/constants.py @@ -0,0 +1,4 @@ +""" +Module containing constants. +""" +API_URL = "https://mitopen.odl.mit.edu/api/v1/courses/?platform=ocw" diff --git a/ocw_oer_export/create_csv.py b/ocw_oer_export/create_csv.py new file mode 100644 index 0000000..6a9eb53 --- /dev/null +++ b/ocw_oer_export/create_csv.py @@ -0,0 +1,195 @@ +""" +Module for creating OER-template CSV file with data extracted from MIT OpenCourseWare API. +""" +import csv +import os +import logging + + +from .client import extract_data_from_api +from .data_handler import extract_data_from_json +from .constants import API_URL +from .utilities import text_cleanup + + +def create_ocw_topic_to_oer_subject_mapping(path=None, file_name=None): + """ + Create a mapping from OCW (OpenCourseWare) topics to OER (Open Educational Resources) subjects. + + This function reads a CSV file containing mappings specified by the arguments. + It creates a dictionary where keys are OCW topics and values are the corresponding OER subjects. + The function does not perform any manipulation on the mapping data. + """ + if path is None: + path = os.path.dirname(__file__) + + if file_name is None: + file_name = "mapping_files/ocw_topic_to_oer_subject.csv" + + file_path = os.path.join(path, file_name) + with open(file_path, newline="", encoding="utf-8") as csvfile: + reader = csv.DictReader(csvfile) + return {row["OCW Topic"]: row["OER Subject"] for row in reader} + + +def get_cr_subjects(ocw_topics_mapping, ocw_course_topics): + """ + Get OER formatted Course Resource Subjects list. + + Since distinct OCW topics can map to the same OER subject, duplicate subject + values are omitted. + """ + oer_subjects_list = [ + ocw_topics_mapping.get(topic["name"]).split("|") + if ocw_topics_mapping.get(topic["name"]) is not None + else [] + for topic in ocw_course_topics + ] + unique_oer_subjects = set( + subject for subjects in oer_subjects_list for subject in subjects + ) + sorted_unique_oer_subjects = sorted(unique_oer_subjects) + return "|".join(sorted_unique_oer_subjects) + + +def get_cr_keywords(list_of_topics_objs): + """Get OER formatted Course Resource Keywords from a list of OCW topic objects.""" + return "|".join(topic["name"] for topic in list_of_topics_objs) + + +def get_cr_authors(list_of_authors_objs): + """Get OER formatted Course Resource Authors list.""" + return "|".join( + f"{author['last_name']}, {author['first_name']}" + for author in list_of_authors_objs + ) + + +def get_cr_educational_use(ocw_course_feature_tags): + """ + Get OER formatted Course Resource Educational Uses list based on OCW Course Feature tags. + + This function analyzes the feature tags of a course and includes: + - 'Curriculum', and 'Instruction' for every course + - 'Assessment' if the feature tags include 'Assignment'. + - 'Professional Development' if the feature tags include 'Instructor Insights'. + """ + tags = ["Curriculum/Instruction"] + assessment_flag = any("Assignment" in tag for tag in ocw_course_feature_tags) + professional_dev_flag = "Instructor Insights" in ocw_course_feature_tags + + if assessment_flag: + tags.append("Assessment") + + if professional_dev_flag: + tags.append("Professional Development") + + return "|".join(tags) + + +def get_cr_accessibility(ocw_course_feature_tags): + """ + Get OER formatted Course Resource Accessibility tags list based on OCW Course Feature tags. + + This function analyzes the feature tags of a course and includes: + - 'Visual', and 'Textual' for every course + - 'Auditory', 'Caption', and 'Transcript' if the feature tags include 'Video'. + """ + tags = ["Visual|Textual"] + video_flag = any("Video" in tag for tag in ocw_course_feature_tags) + + if video_flag: + tags.append("Auditory|Caption|Transcript") + + return "|".join(tags) + + +def get_description_in_plain_text(description): + """Get Course Resource plain text description by cleaning up markdown and HTML.""" + cleaned_description = text_cleanup(description) + return cleaned_description + + +def transform_single_course(course, ocw_topics_mapping): + """Transform a single course according to OER template.""" + return { + "CR_TITLE": course["title"], + "CR_URL": course["runs"][0]["url"], + "CR_MATERIAL_TYPE": "Full Course", + "CR_Media_Formats": "Text/HTML", + "CR_SUBLEVEL": "null", + "CR_ABSTRACT": get_description_in_plain_text(course["runs"][0]["description"]), + "CR_LANGUAGE": "en", + "CR_COU_TITLE": "Creative Commons Attribution Non Commercial Share Alike 4.0", + "CR_PRIMARY_USER": "student|teacher", + "CR_SUBJECT": get_cr_subjects(ocw_topics_mapping, course["topics"]), + "CR_KEYWORDS": get_cr_keywords(course["topics"]), + "CR_AUTHOR_NAME": get_cr_authors(course["runs"][0]["instructors"]), + "CR_PROVIDER": "MIT", + "CR_PROVIDER_SET": "MIT OpenCourseWare", + "CR_COU_URL": "https://creativecommons.org/licenses/by-nc-sa/4.0/", + "CR_COU_COPYRIGHT_HOLDER": "MIT", + "CR_EDUCATIONAL_USE": get_cr_educational_use(course["resource_content_tags"]), + "CR_ACCESSIBILITY": get_cr_accessibility(course["resource_content_tags"]), + } + + +def transform_data(data, ocw_topics_mapping): + """Transform all courses into OER template.""" + return [ + course + for course in ( + transform_single_course(course, ocw_topics_mapping) for course in data + ) + if course is not None + ] + + +def create_csv( + source="api", input_file="ocw_api_data.json", output_file="ocw_oer_export.csv" +): + """Create a CSV file from either the MIT OpenCourseWare API or a locally stored JSON file.""" + api_data_json = {} + + if source == "api": + api_data_json = extract_data_from_api(api_url=API_URL) + + elif source == "json": + api_data_json = extract_data_from_json(input_file) + + else: + raise ValueError("Invalid source. Use 'api' or 'json'.") + + ocw_topics_mapping = create_ocw_topic_to_oer_subject_mapping() + transformed_data = transform_data(api_data_json, ocw_topics_mapping) + fieldnames = [ + "CR_TITLE", + "CR_URL", + "CR_MATERIAL_TYPE", + "CR_Media_Formats", + "CR_SUBLEVEL", + "CR_ABSTRACT", + "CR_LANGUAGE", + "CR_COU_TITLE", + "CR_PRIMARY_USER", + "CR_SUBJECT", + "CR_KEYWORDS", + "CR_AUTHOR_NAME", + "CR_PROVIDER", + "CR_PROVIDER_SET", + "CR_COU_URL", + "CR_COU_COPYRIGHT_HOLDER", + "CR_EDUCATIONAL_USE", + "CR_ACCESSIBILITY", + ] + with open(output_file, "w", newline="", encoding="utf-8") as csv_file: + writer = csv.DictWriter(csv_file, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(transformed_data) + + current_dir = os.getcwd() + logging.info( + "CSV file '%s' successfully created at directory: %s", + output_file, + current_dir, + ) diff --git a/ocw_oer_export/create_json.py b/ocw_oer_export/create_json.py new file mode 100644 index 0000000..74ac91d --- /dev/null +++ b/ocw_oer_export/create_json.py @@ -0,0 +1,22 @@ +""" +Module to interact with MIT OpenCourseWare API and write data to a JSON file. +""" +import json +import logging + +from .constants import API_URL +from .client import extract_data_from_api + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +def create_json(output_file="ocw_api_data.json"): + """Fetches data from MIT OpenCourseWare API and writes it to a JSON file.""" + api_data = extract_data_from_api(api_url=API_URL) + try: + with open(output_file, "w", encoding="utf-8") as json_file: + json.dump(api_data, json_file, ensure_ascii=False, indent=4) + logger.info("Data saved to %s at present directory.", output_file) + except IOError as e: + logger.error("Error saving data to JSON: %s", e) diff --git a/ocw_oer_export/data_handler.py b/ocw_oer_export/data_handler.py new file mode 100644 index 0000000..eec3fde --- /dev/null +++ b/ocw_oer_export/data_handler.py @@ -0,0 +1,35 @@ +""" +Module for extracting and loading data to/from JSON file +""" +import json +import logging + + +def extract_data_from_json(file_path): + """Extract data from a JSON file.""" + try: + with open(file_path, "r", encoding="utf-8") as json_file: + data = json.load(json_file) + logging.info("Data successfully extracted from %s.", file_path) + return data + except FileNotFoundError as exc: + raise FileNotFoundError(f"{file_path} not found.") from exc + + +def load_data_to_json(data, file_path): + """Save data to a JSON file.""" + + with open(file_path, "r", encoding="utf-8") as json_file: + json.dump(data, json_file) + logging.info("Data successfully saved: %s.", file_path) + + +def extract_data_from_file(file_path): + """Read file using readlines()""" + try: + with open(file_path, "r", encoding="utf-8") as file: + data = file.readlines() + logging.info("Data successfully extracted from %s.", file_path) + return data + except FileNotFoundError as exc: + raise FileNotFoundError(f"{file_path} not found.") from exc diff --git a/ocw_oer_export/mapping_files/ocw_topic_to_oer_subject.csv b/ocw_oer_export/mapping_files/ocw_topic_to_oer_subject.csv new file mode 100644 index 0000000..18b41c0 --- /dev/null +++ b/ocw_oer_export/mapping_files/ocw_topic_to_oer_subject.csv @@ -0,0 +1,329 @@ +OCW Topic,OER Subject,OER Keyword +Business,Business and Communication, +Accounting,Accounting, +Business Ethics,Business Ethicates, +Entrepreneurship,Business and Communication,Entrepreneurship +Finance,Finance, +Globalization,Business and Communication,Globalization +Health Care Management,"Management|Health, Medicine and Nursing",Health Care Management +Industrial Relations and Human Resource Management,Management|Public Relations,Industrial Relations|Human Resource Management +Information Technology,Business and Communication,Information Technology +Innovation,Business and Communication,Innovation +Leadership,Management,Leadership +Management,Management, +Marketing,Marketing, +Operations Management,Management,Operations Management +Organizational Behavior,Business and Communication,Organizational Behavior +Real Estate,Business and Communication,Real Estate +Supply Chain Management,Management,Supply Chain Management +Energy,Environmental Studies|Environmental Science,Energy +Buildings,Environmental Studies,Buildings +Climate,Environmental Studies|Environmental Science,Climate +Combustion,Environmental Studies|Environmental Science,Combustion +Electricity,Environmental Studies|Environmental Science,Electricity +Fossil Fuels,Environmental Studies|Environmental Science,Fossil Fuels +Fuel Cells,Environmental Studies|Environmental Science,Fuel Cells +Hydrogen and Alternatives,Environmental Studies|Environmental Science,Hydrogen and Alternatives +Nuclear,Environmental Studies|Environmental Science,Nuclear +Renewables,Environmental Studies|Environmental Science,Renewable Energy +Technology,Environmental Studies|Environmental Science,Energy Technology +Transportation,Environmental Studies,Transportation +Engineering,Engineering, +Aerospace Engineering,Engineering,Aerospace Engineering +Aerodynamics,Engineering,Aerodynamics +Astrodynamics,Engineering|Astronomy,Astrodynamics +Avionics,Engineering,Avionics +Bioastronautics,Engineering|Astronomy,Bioastronautics +Guidance and Control Systems,Engineering,Guidance and Control Systems +Materials Selection,Engineering,Materials Selection +Propulsion Systems,Engineering,Propulsion Systems +Structural Mechanics,Engineering,Structural Mechanics +Biological Engineering,Engineering|Biology,Biological Engineering +Biomechanics,Engineering|Biology,Biomechanics +Biostatistics,Engineering|Biology,Biostatistics +Biotechnology,Engineering|Biology,Biotechnology +Cell and Tissue Engineering,Engineering|Biology,Cell Engineering|Tissue Engineering +Computational Biology,Engineering|Biology,Computational Biology +Synthetic Biology,Engineering|Biology,Synthetic Biology +Chemical Engineering,Engineering|Chemistry,Chemical Engineering +Molecular Engineering,Engineering|Chemistry,Molecular Engineering +Polymers,Engineering|Chemistry,Polymers +Process Control Systems,Engineering|Chemistry,Process Control Systems +Separation Processes,Engineering|Chemistry,Separation Processes +Transport Processes,Engineering|Chemistry,Transport Processes +Civil Engineering,Engineering,Civil Engineering +Construction Management,Engineering|Management,Construction Management +Geotechnical Engineering,Engineering,Geotechnical Engineering +Structural Engineering,Engineering,Structural Engineering +Surveying,Engineering,Surveying +Transportation Engineering,Engineering,Transportation Engineering +Computer Science,Computer Science, +Algorithms and Data Structures,Computer Science,Algorithms|Data Structures +Artificial Intelligence,Computer Science,Artificial Intelligence +Computer Design and Engineering,Computer Science,Computer Design| Computer Engineering +Computer Networks,Computer Science,Computer Networks +Cryptography,Computer Science,Cryptography +Data Mining,Computer Science,Data Mining +Graphics and Visualization,Computer Science,Graphics|Visualization +Human-Computer Interfaces,Computer Science,Human-Computer Interfaces +Operating Systems,Computer Science,Operating Systems +Programming Languages,Computer Science,Programming Languages +Software Design and Engineering,Computer Science,Software Design|Software Engineering +Theory of Computation,Computer Science,Theory of Computation +Electrical Engineering,Engineering|Electronic Technology,Electrical Engineering +Digital Systems,Engineering|Electronic Technology,Digital Systems +Electric Power,Engineering|Electronic Technology,Electric Power +Electronics,Engineering|Electronic Technology,Electronics +Robotics and Control Systems,Engineering|Electronic Technology,Robotics and Control Systems +Signal Processing,Engineering|Electronic Technology,Signal Processing +Telecommunications,Engineering|Electronic Technology,Telecommunications +Environmental Engineering,Engineering|Environmental Science,Environmental Engineering +Aquatic Sciences and Water Quality Control,Engineering|Environmental Science,Aquatic Sciences|Water Quality Control +Environmental Management,Engineering|Environmental Science,Environmental Management +Hydrodynamics and Coastal Engineering,Engineering|Environmental Science,Hydrodynamics|Coastal Engineering +Hydrology and Water Resource Systems,Engineering|Environmental Science|Hydrology,Water Resource Systems +Materials Science and Engineering,Engineering,Materials Science and Engienering +Composite Materials,Engineering,Composite Materials +Electronic Materials,Engineering,Electronic Materials +Metallurgical Engineering,Engineering,Metallurgical Engineering +Polymeric Materials,Engineering,Polymeric Materials +Mechanical Engineering,Engineering,Mechanical Engineering +Dynamics and Control,Engineering,Dynamics and Control +Fluid Mechanics,Engineering,Fluid Mechanics +Mechanical Design,Engineering,Mechanical Design +Microtechnology,Engineering,Microtechnology +Solid Mechanics,Engineering,Solid Mechanics +Nanotechnology,Engineering,Nanotechnology +Nuclear Engineering,Engineering|Environmental Science,Nuclear Engineering +Nuclear Materials,Engineering|Environmental Science,Nuclear Materials +"Nuclear Systems, Policy, and Economics",Engineering|Economics|Political Science,Nuclear Systems|Nuclear Policy|Nuclear Economics +Radiological Engineering,Engineering|Environmental Science,Radiological Engineering +Ocean Engineering,Engineering|Oceanography,Ocean Engineering +Hydrodynamics,Engineering|Oceanography,Hydrodynamics +Ocean Exploration,Engineering|Oceanography,Ocean Exploration +Ocean Structures,Engineering|Oceanography,Ocean Structures +Oceanic Pollution Control,Engineering|Oceanography,Oceanic Pollution Control +Systems Engineering,Engineering,Systems Engineering +Computational Modeling and Simulation,Engineering|Mathematics,Computational Modeling|Computational Simulation +Computational Science and Engineering,Engineering|Computer Science,Computational Science|Computational Engineering +Numerical Simulation,Engineering|Mathematics,Numerical Simulation +Systems Design,Engineering,Systems Design +Systems Optimization,Engineering,Systems Optimization +Fine Arts,Arts and Humanities,Fine Arts +Architecture,Architecture and Design, +Architectural Design,Architecture and Design, +Architectural Engineering,Architecture and Design,Architectural Engineering +Architectural History and Criticism,Architecture and Design,Architectural History| Architectural Criticism +Environmental Design,Architecture and Design,Environmental Design +Religious Architecture,Architecture and Design,Religious Architecture +Art History,Art History, +Game Design,Graphic Arts,Game Design +Media Studies,Graphic Arts,Media Studies +Digital Media,Graphic Arts|Graphic Design,Digital Media +Music,Performing Arts,Music +Music History,Performing Arts,Music History +Music Performance,Performing Arts,Music Performance +Music Theory,Performing Arts,Music Theory +Performance Arts,Performing Arts, +Dance,Performing Arts,Dance +Theater,Performing Arts,Theater +Theatrical Design,Performing Arts,Theatrical Design +Visual Arts,Visual Arts, +Film and Video,Visual Arts|Film and Music Production,Film and Video +Graphic Design,Visual Arts|Graphic Arts,Graphic Design +Photography,Visual Arts,Photography +Health and Medicine,"Health, Medicine and Nursing", +Anatomy and Physiology,"Health, Medicine and Nursing|Anatomy/Physiology", +Biomedical Enterprise,"Health, Medicine and Nursing",Biomedical Enterprise +Biomedical Instrumentation,"Health, Medicine and Nursing",Biomedical Instrumentation +Biomedical Signal and Image Processing,"Health, Medicine and Nursing",Biomedical Signal Processing| Biomedical Image Processing +Biomedicine,"Health, Medicine and Nursing",Biomedicine +Cancer,"Health, Medicine and Nursing",Cancer +Cellular and Molecular Medicine,"Health, Medicine and Nursing",Cellular Medicine| Molecular Medicine +Epidemiology,"Health, Medicine and Nursing",Epidemiology +Functional Genomics,"Health, Medicine and Nursing",Functional Genomics +Health and Exercise Science,"Health, Medicine and Nursing",Health and Exercise Science +Immunology,"Health, Medicine and Nursing",Immunology +Medical Imaging,"Health, Medicine and Nursing",Medical Imaging +Mental Health,"Health, Medicine and Nursing",Mental Health +Pathology and Pathophysiology,"Health, Medicine and Nursing",Pathology| Pathophysiology +Pharmacology and Toxicology,"Health, Medicine and Nursing",Pharmacology|Toxicology +Physical Education and Recreation,"Health, Medicine and Nursing",Physical Education and Recreation +Public Health,"Health, Medicine and Nursing",Public Health +Sensory-Neural Systems,"Health, Medicine and Nursing",Sensory-Neural Systems +Social Medicine,"Health, Medicine and Nursing",Social Medicine +Spectroscopy,"Health, Medicine and Nursing",Spectroscopy +Speech Pathology,"Health, Medicine and Nursing",Speech Pathology +Humanities,Arts and Humanities, +History,History, +African History,World History,African History +American History,U.S. History, +Ancient History,Ancient History, +Asian History,World History,Asian History +Comparative History,History,Comparative History +European History,World History,European History +Historical Methods,History,Historical Methods +Historiography,History,Historiography +History of Science and Technology,History,History of Science|History of Technology +Intellectual History,History,Intellectual History +Jewish History,World History,Jewish History +Latin American History,World History,Latin American History +Medieval History,Ancient History,Medieval History +Middle Eastern History,World History,Middle Eastern History +Military History,History,Military History +Modern History,History,Modern History +World History,World History, +Language,Languages, +Chinese,Languages,Chinese +English as a Second Language,Language Education (ESL), +French,Languages,French +German,Languages,German +Japanese,Languages,Japanese +Portuguese,Languages,Portuguese +Spanish,Languages,Spanish +Linguistics,Linguistics, +Phonology,Linguistics,Phonology +Semantics,Linguistics,Semantics +Syntax,Linguistics,Syntax +Literature,Literature, +Academic Writing,Literature,Academic Writing +American Literature,Literature|Reading Literature,American Literature +Biography,Literature|Reading Literature,Biography +Classics,Literature|Reading Literature,Classics +Comedy,Literature|Reading Literature,Comedy +Comparative Literature,Literature|Reading Literature,Comparative Literature +Creative Writing,Literature|Composition and Rehtoric,Creative Writing +Criticism,Literature,Criticism +Drama,Literature|Reading Literature,Drama +Fiction,Literature|Reading Literature,Fiction +International Literature,Literature|Reading Literature,International Literature +Nonfiction Prose *Periodic Literature,Literature|Reading Literature,Nonfiction Prose +Poetry,Literature|Reading Literature,Poetry +Rhetoric,Literature|Composition and Rehtoric, +Technical Writing,Literature|Composition and Rehtoric,Technical Writing +Philosophy,Philosophy, +Epistemology,Philosophy,Epistemology +Ethics,Philosophy,Ethics +Logic,Philosophy,Logic +Metaphysics,Philosophy,Metaphysics +Philosophy of Language,Philosophy,Philosophy of Language +Political Philosophy,Philosophy,Political Philosophy +Religion,Religious Studies, +Mathematics,Mathematics, +Algebra and Number Theory,Algebra,Number Theory +Applied Mathematics,Mathematics,Applied Mathematics +Calculus,Calculus, +Computation,Mathematics,Computation +Differential Equations,Mathematics,Differential Equations +Linear Algebra,Algebra,Linear Algebra +Mathematical Analysis,Mathematics,Mathematical Analysis +Mathematical Logic,Mathematics,Mathematical Logic +Probability and Statistics,Statistics and Probability, +Topology and Geometry,Geometry,Topology +Science,Physical Science, +Biology,Biology, +Biochemistry,Biology,Biochemistry +Biophysics,Biology,Biophysics +Cell Biology,Biology,Cell biology +Computation and Systems Biology,Biology,Computation and Systems Biology +Developmental Biology,Biology,Developmental Biology +Ecology,Ecology, +Genetics,Genetics, +Microbiology,Biology,Microbiology +Molecular Biology,Biology,Molecular Biology +Neurobiology,Biology,Neurobiology +Neuroscience,Biology,Neuroscience +Proteomics,Biology,Proteomics +Stem Cells,Biology,Stem Cells +Structural Biology,Biology,Structural Biology +Virology,Biology,Virology +Chemistry,Chemistry, +Analytical Chemistry,Chemistry,Analytical Chemistry +Inorganic Chemistry,Chemistry,Inorganic Chemistry +Organic Chemistry,Chemistry,Organic Chemistry +Physical Chemistry,Chemistry,Physical Chemistry +Cognitive Science,Life Science,Cognitive Science +Earth Science,Physical Science|Atmospheric Science,Earth Science +Atmospheric Science,Atmospheric Science, +Climate Studies,Physical Science,Climate Studies +Environmental Science,Environmental Science, +Geobiology,Physical Science,Geobiology +Geochemistry,Physical Science,Geochemistry +Geology,Physical Geology, +Geophysics,Physical Science,Geophysics +Oceanography,Oceanography, +Planetary Science,Physical Science|Atmospheric Science,Planetary Science +Sustainability,Physical Science,Sustainability +Physics,Physics, +Astrophysics,Physics,Astrophysics +"Atomic, Molecular, Optical Physics",Physics,Atomic Physics|Molecular Physics|Optical Physics +Classical Mechanics,Physics,Classical Mechanics +Condensed Matter Physics,Physics,Condensed Matter Physics +Electromagnetism,Physics,Electromagnetism +High Energy Physics,Physics,High Energy Physics +Nuclear Physics,Physics,Nuclear Physics +Particle Physics,Physics,Particle Physics +Quantum Mechanics,Physics,Quantum Mechanics +Relativity,Physics,Relativity +Theoretical Physics,Physics,Theoretical Physics +Thermodynamics,Physics,Thermodynamics +Social Science,Social Science, +Anthropology,Anthropology, +Biological Anthropology,Anthropology,Biological Anthropology +Cultural Anthropology,Anthropology,Cultural Anthropology +Ethnography,Anthropology,Ethnography +Social Anthropology,Anthropology,Social Anthropology +Archaeology,Archaeology, +Communication,Communication, +Economics,Economics, +Developmental Economics,Economics,Developmental Economics +Econometrics,Economics,Econometrics +Financial Economics,Economics,Financial Economics +Industrial Organization,Economics,Industrial Organization +International Development,Economics,International Development +International Economics,Economics,International Economics +Labor Economics,Economics,Labor Economics +Macroeconomics,Economics,Macroeconomics +Microeconomics,Economics,Microeconomics +Political Economy,Economics|Political Science,Political Economy +Public Economics,Economics,Public Economics +Game Theory,Mathematics|Information Science|Computer Science,Game Theory +Geography,Physical Geography, +Legal Studies,Law,Legal Studies +Political Science,Political Science, +American Politics,Political Science,American Politics|U.S. Politics +Comparative Politics,Political Science,Comparative Politics +International Relations,Political Science,International Relations +Psychology,Psychology, +Public Administration,Political Science,Public Administration +Environmental Policy,Political Science|Environmental Studies,Environmental Policy +Military Studies,Political Science,Military Studies +Public Policy,Political Science,Public Policy +Regional Politics,Political Science,Regional Politics +Science and Technology Policy,Political Science,Science and Technology Policy +Security Studies,Political Science,Security Studies +Social Welfare,Political Science,Social Welfare +Sociology,Sociology, +Community Development,Sociology,Community Development +Social Justice,Sociology,Social Justics +Urban Studies,Social Science,Urban Studies +Housing Development,Social Science,Housing Development +Regional Planning,Social Science,Regional Planning +Transportation Planning,Social Science,Transportation Planning +Urban Planning,Social Science,Urban Planning +Society,Social Science, +African-American Studies,Ethnic Studies,African-American Studies +Asian Studies,World Cultures,Asian Studies +European and Russian Studies,World Cultures,European and Russian Studies +Gender Studies,Social Science|Women’s Studies,Gender Studies +Global Poverty,Cultural Geography,Global Poverty +Indigenous Studies,Ethnic Studies|Cultural Geography,Indigenous Studies +Latin and Caribbean Studies,World Cultures,Latin and Caribbean Studies +Middle Eastern Studies,World Cultures,Middle Eastern Studies +The Developing World,Cultural Geography,Developing World +Women’s Studies,Women’s Studies, +Teaching and Education,Education,Teaching +Curriculum and Teaching,Education,Curriculum|Teaching +Education Policy,Education,Education Policy +Educational Technology,Educational Technology, +Higher Education,Higher Education, diff --git a/ocw_oer_export/utilities/__init__.py b/ocw_oer_export/utilities/__init__.py new file mode 100644 index 0000000..60193e2 --- /dev/null +++ b/ocw_oer_export/utilities/__init__.py @@ -0,0 +1,4 @@ +__all__ = ["text_cleanup", "delete_file"] + +from .text_cleanup import text_cleanup +from .delete_file import delete_file diff --git a/ocw_oer_export/utilities/delete_file.py b/ocw_oer_export/utilities/delete_file.py new file mode 100644 index 0000000..1a6d9c6 --- /dev/null +++ b/ocw_oer_export/utilities/delete_file.py @@ -0,0 +1,10 @@ +""" +Module for deleting a file. +""" +import os + + +def delete_file(file_path): + """Delete a file if it exists""" + if os.path.exists(file_path): + os.remove(file_path) diff --git a/ocw_oer_export/utilities/text_cleanup.py b/ocw_oer_export/utilities/text_cleanup.py new file mode 100644 index 0000000..7842cba --- /dev/null +++ b/ocw_oer_export/utilities/text_cleanup.py @@ -0,0 +1,64 @@ +""" +Module for cleaning up HTML and Markdown mixed text coming out of MIT's OCW Studio's CKEditor. +""" + +import re +from io import StringIO +from markdown import Markdown + + +def cleanup_curly_brackets(text): + """ + Remove content within curly brackets, including the brackets themselves, from the input string. + Also remove any lines that become empty as a result of this removal. + """ + pattern = re.compile(r"\{\{.*?\}\}\n?") + cleaned_text = re.sub(pattern, "", text) + return cleaned_text + + +def html_to_text(html): + """Remove HTML tags from an HTML string.""" + pattern = re.compile("<.*?>") + return re.sub(pattern, "", html) + + +def create_markdown_converter(): + """Create and configure a Markdown converter for plain text output.""" + + def unmark_element(element, stream=None): + """Helper function to recursively extract text from Markdown elements.""" + if stream is None: + stream = StringIO() + if element.text: + stream.write(element.text) + for sub in element: + unmark_element(sub, stream) + if element.tail: + stream.write(element.tail) + return stream.getvalue() + + # patching Markdown + Markdown.output_formats["plain"] = unmark_element + markdown_converter = Markdown(output_format="plain") + markdown_converter.stripTopLevelTags = False + return markdown_converter + + +def markdown_to_text(markdown): + """Convert Markdown to plain text using the markdown_converter.""" + markdown_converter = create_markdown_converter() + return markdown_converter.convert(markdown) + + +def text_cleanup(text): + """ + Perform text cleanup by: + 1. Converting Markdown to text, + 2. Removing HTML tags, + 3. Cleaning up curly brackets. + """ + stripped_markdown = markdown_to_text(text) + stripped_html = html_to_text(stripped_markdown) + cleaned_text = cleanup_curly_brackets(stripped_html) + return cleaned_text diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..f1bfca4 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ['setuptools>=42'] +build-backend = 'setuptools.build_meta' diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..49d860a --- /dev/null +++ b/requirements.txt @@ -0,0 +1,21 @@ +certifi==2023.11.17 +cfgv==3.4.0 +charset-normalizer==3.3.2 +decorator==5.1.1 +distlib==0.3.8 +filelock==3.13.1 +identify==2.5.33 +idna==3.6 +importlib-metadata==7.0.1 +Markdown==3.5.1 +nodeenv==1.8.0 +platformdirs==4.1.0 +pre-commit==3.6.0 +py==1.11.0 +PyYAML==6.0.1 +requests==2.31.0 +retry==0.9.2 +tqdm==4.66.1 +urllib3==2.1.0 +virtualenv==20.25.0 +zipp==3.17.0 diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..ee3ae19 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,32 @@ +[metadata] +name = ocw_oer_export +version = 1.0.0 +author = X +author_email = Y +description = A package to export MIT OpenCourseWare (OCW) courses metadata to CSV format, compatible with OER Commons import requirements. +long_description = file: README.md +long_description_content_type = text/markdown +url = https://github.com/mitodl/ocw_oer_export +license = BSD-3-Clause +classifiers = + Programming Language :: Python :: 3 + License :: OSI Approved :: BSD License + Operating System :: OS Independent + +[options] +python_requires = >=3.9.6 +install_requires = + requests>=2.31.0 + retry>=0.9.2 + tqdm>=4.66.1 + Markdown>=3.5.1 + pre-commit>=3.6.0 + +packages = find: + +[options.entry_points] +console_scripts = + ocw-oer-export = ocw_oer_export.cli:main + +[options.package_data] +ocw_oer_export = mapping_files/* diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..6068493 --- /dev/null +++ b/setup.py @@ -0,0 +1,3 @@ +from setuptools import setup + +setup() diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/expected_courses.csv b/tests/expected_courses.csv new file mode 100644 index 0000000..41e8b83 --- /dev/null +++ b/tests/expected_courses.csv @@ -0,0 +1,25 @@ +CR_TITLE,CR_URL,CR_MATERIAL_TYPE,CR_Media_Formats,CR_SUBLEVEL,CR_ABSTRACT,CR_LANGUAGE,CR_COU_TITLE,CR_PRIMARY_USER,CR_SUBJECT,CR_KEYWORDS,CR_AUTHOR_NAME,CR_PROVIDER,CR_PROVIDER_SET,CR_COU_URL,CR_COU_COPYRIGHT_HOLDER,CR_EDUCATIONAL_USE,CR_ACCESSIBILITY +Hands-On Introduction to Electrical Engineering Lab Skills,https://ocw.mit.edu/courses/6-091-hands-on-introduction-to-electrical-engineering-lab-skills-january-iap-2008,Full Course,Text/HTML,null,"This course introduces students to both passive and active electronic components (op-amps, 555 timers, TTL digital circuits). Basic analog and digital circuits and theory of operation are covered. The labs allow the students to master the use of electronic instruments and construct and/or solder several circuits. The labs also reinforce the concepts discussed in class with a hands-on approach and allow the students to gain significant experience with electrical instruments such as function generators, digital multimeters, oscilloscopes, logic analyzers and power supplies. In the last lab, the students build an electronic circuit that they can keep. The course is geared to freshmen and others who want an introduction to electronics circuits. +This course is offered during the Independent Activities Period (IAP), which is a special 4-week term at MIT that runs from the first week of January until the end of the month.",en,Creative Commons Attribution Non Commercial Share Alike 4.0,student|teacher,Electronic Technology|Engineering,Engineering|Electrical Engineering,"Hom, Gim",MIT,MIT OpenCourseWare,https://creativecommons.org/licenses/by-nc-sa/4.0/,MIT,Curriculum/Instruction,Visual|Textual +Thermodynamics and Kinetics of Materials,https://ocw.mit.edu/courses/3-205-thermodynamics-and-kinetics-of-materials-fall-2006,Full Course,Text/HTML,null,"This course explores materials and materials processes from the perspective of thermodynamics and kinetics. The thermodynamics aspect includes laws of thermodynamics, solution theory and equilibrium diagrams. The kinetics aspect includes diffusion, phase transformations, and the development of microstructure.",en,Creative Commons Attribution Non Commercial Share Alike 4.0,student|teacher,Chemistry|Engineering|Physical Science|Physics,Engineering|Science|Chemistry|Thermodynamics|Materials Science and Engineering|Physical Chemistry|Physics,"Allen, Samuel|Eagar, Thomas",MIT,MIT OpenCourseWare,https://creativecommons.org/licenses/by-nc-sa/4.0/,MIT,Curriculum/Instruction,Visual|Textual +Communicating in American Culture(s),https://ocw.mit.edu/courses/21g-221-communicating-in-american-culture-s-spring-2019,Full Course,Text/HTML,null,"In Communicating in American Culture(s), bilingual students examine how various aspects of American culture—history, geography, institutions, traditions, values—have shaped dominant Anglo-American communication norms and responses to critical events in the world. In addition, you can expect to practice and strengthen your analytical and communication skills in a carefully scaffolded manner, starting with frequent short writing and speaking tasks and progressing to longer, more formal tasks.",en,Creative Commons Attribution Non Commercial Share Alike 4.0,student|teacher,Anthropology|Arts and Humanities|Communication|Language Education (ESL)|Languages|Social Science,Social Science|Humanities|Communication|Language|Anthropology|Cultural Anthropology|English as a Second Language,"Dunphy, Jane",MIT,MIT OpenCourseWare,https://creativecommons.org/licenses/by-nc-sa/4.0/,MIT,Curriculum/Instruction|Assessment,Visual|Textual +Turning Evolutionary Dials: Directed Evolution Techniques for Climate Change and Beyond,https://ocw.mit.edu/courses/7-341-turning-evolutionary-dials-directed-evolution-techniques-for-climate-change-and-beyond-spring-2022,Full Course,Text/HTML,null,"This course will cover the many ways in which we have realized evolution in the laboratory toward functional biomolecules, such as protein and nucleic-acid-based therapeutics, enzymes that catalyze production of synthetic drugs, and carbon-dioxide capture molecules to lessen the impact of climate change. Students will both become familiar with the field of directed molecular evolution and learn how to critically analyze primary research papers, design research experiments, and present data relating to molecular biology and evolution. The importance of directed evolution in biomedical and biotechnological careers, both academic and industrial, will be highlighted. +This course is one of many Advanced Undergraduate Seminars offered by the Biology Department at MIT. These seminars are tailored for students with an interest in using primary research literature to discuss and learn about current biological research in a highly interactive setting. Many instructors of the Advanced Undergraduate Seminars are postdoctoral scientists with a strong interest in teaching.",en,Creative Commons Attribution Non Commercial Share Alike 4.0,student|teacher,Biology|Physical Science,Biochemistry|Science|Biology|Molecular Biology|Cell Biology,"Kizer, Megan|Wilson, Robbie",MIT,MIT OpenCourseWare,https://creativecommons.org/licenses/by-nc-sa/4.0/,MIT,Curriculum/Instruction|Assessment,Visual|Textual +"Probability And Its Applications To Reliability, Quality Control, And Risk Assessment",https://ocw.mit.edu/courses/22-38-probability-and-its-applications-to-reliability-quality-control-and-risk-assessment-fall-2005,Full Course,Text/HTML,null,"This course covers interpretations of the concept of probability. Topics include basic probability rules; random variables and distribution functions; functions of random variables; and applications to quality control and the reliability assessment of mechanical/electrical components, as well as simple structures and redundant systems. The course also considers elements of statistics; Bayesian methods in engineering; methods for reliability and risk assessment of complex systems (event-tree and fault-tree analysis, common-cause failures, human reliability models); uncertainty propagation in complex systems (Monte Carlo methods, Latin Hypercube Sampling); and an introduction to Markov models. Examples and applications are drawn from nuclear and other industries, waste repositories, and mechanical systems.",en,Creative Commons Attribution Non Commercial Share Alike 4.0,student|teacher,Engineering|Mathematics|Statistics and Probability,Engineering|Mathematics|Probability and Statistics|Systems Engineering,"Golay, Michael",MIT,MIT OpenCourseWare,https://creativecommons.org/licenses/by-nc-sa/4.0/,MIT,Curriculum/Instruction,Visual|Textual +Strategic HR Management,https://ocw.mit.edu/courses/15-660-strategic-hr-management-spring-2003,Full Course,Text/HTML,null,"This course is about both the design and execution of human resource management strategies. This course has two central themes: (1) How to think systematically and strategically about aspects of managing the organization's human assets, and (2) What really needs to be done to implement these policies and to achieve competitive advantage. It adopts the perspective of a general manager and addresses human resource topics (including reward systems, performance management, high-performance human resource systems, training and development, recruitment, retention, equal employment opportunity laws, work-force diversity, and union-management relationships) from a strategic perspective.",en,Creative Commons Attribution Non Commercial Share Alike 4.0,student|teacher,Business and Communication|Management|Public Relations,Industrial Relations and Human Resource Management|Business,"Burton, Diane|Osterman, Paul",MIT,MIT OpenCourseWare,https://creativecommons.org/licenses/by-nc-sa/4.0/,MIT,Curriculum/Instruction,Visual|Textual +Linguistic Phonetics,https://ocw.mit.edu/courses/24-915-linguistic-phonetics-fall-2015,Full Course,Text/HTML,null,"This course is about the study of speech sounds; how we produce and perceive them and their acoustic properties. Topics include the influence of the production and perception systems on phonological patterns and sound change, students learn acoustic analysis and experimental techniques. Students taking the graduate version complete different assignments.",en,Creative Commons Attribution Non Commercial Share Alike 4.0,student|teacher,Arts and Humanities|Linguistics,Linguistics|Phonology|Humanities,"Flemming, Edward",MIT,MIT OpenCourseWare,https://creativecommons.org/licenses/by-nc-sa/4.0/,MIT,Curriculum/Instruction|Assessment,Visual|Textual +"Gender, Power, Leadership, and the Workplace",https://ocw.mit.edu/courses/wgs-s10-gender-power-leadership-and-the-workplace-spring-2014,Full Course,Text/HTML,null,"This course will provide students with an analytic framework to understand the roles that gender, race, and class play in defining and determining access to leadership and power in the U.S., especially in the context of the workplace. We will explore women and men in leadership positions within the corporate, political and non-profit sectors, with attention to the roles of women of color and immigrant women within this context. We will also look at specific policies such as affirmative action, parental leave, child-care policy, and working-time policies and the role they play–or could play–in achieving parity. We will further investigate ways in which these policies address gender, racial, and class inequities, and think critically about mechanisms for change. The course will be highly interactive, and will combine texts, theater, videos and visual arts.",en,Creative Commons Attribution Non Commercial Share Alike 4.0,student|teacher,Business and Communication|Management|Political Science|Social Science|Women’s Studies,Social Science|Business|Political Science|Society|Women's Studies|Leadership|Gender Studies|Public Policy|Social Welfare|Public Administration|American Politics,"Fried, Mindy",MIT,MIT OpenCourseWare,https://creativecommons.org/licenses/by-nc-sa/4.0/,MIT,Curriculum/Instruction|Assessment|Professional Development,Visual|Textual +Theory of Probability,https://ocw.mit.edu/courses/18-175-theory-of-probability-spring-2014,Full Course,Text/HTML,null,"This course covers topics such as sums of independent random variables, central limit phenomena, infinitely divisible laws, Levy processes, Brownian motion, conditioning, and martingales.",en,Creative Commons Attribution Non Commercial Share Alike 4.0,student|teacher,Mathematics|Statistics and Probability,Probability and Statistics|Mathematics|Mathematical Analysis,"Sheffield, Scott",MIT,MIT OpenCourseWare,https://creativecommons.org/licenses/by-nc-sa/4.0/,MIT,Curriculum/Instruction,Visual|Textual +Entrepreneurial Marketing,https://ocw.mit.edu/courses/15-835-entrepreneurial-marketing-spring-2002,Full Course,Text/HTML,null,"This course clarifies key marketing concepts, methods, and strategic issues relevant for start-up and early-stage entrepreneurs. At this course, there are two major questions: + +Marketing Question: What and how am I selling to whom? +New Venture Question: How do I best leverage my limited marketing recourses? + +Specifically, this course is designed to give students a broad and deep understanding of such topics as: + +What are major strategic constraints and issues confronted by entrepreneurs today? +How can one identify and evaluate marketing opportunities? +How do entrepreneurs achieve competitive advantages given limited marketing resources? +What major marketing/sales tools are most useful in an entrepreneurial setting? + +Because there is no universal marketing solution applicable to all entrepreneurial ventures, this course is designed to help students develop a flexible way of thinking about marketing problems in general.",en,Creative Commons Attribution Non Commercial Share Alike 4.0,student|teacher,Business and Communication|Marketing,Marketing|Entrepreneurship|Business,"Kim, Jin",MIT,MIT OpenCourseWare,https://creativecommons.org/licenses/by-nc-sa/4.0/,MIT,Curriculum/Instruction|Assessment,Visual|Textual diff --git a/tests/sample_courses.json b/tests/sample_courses.json new file mode 100644 index 0000000..1a6053e --- /dev/null +++ b/tests/sample_courses.json @@ -0,0 +1,1147 @@ +[ + { + "id": 3584, + "topics": [ + { + "id": 32, + "name": "Engineering" + }, + { + "id": 42, + "name": "Electrical Engineering" + } + ], + "offered_by": { + "code": "ocw", + "name": "OCW" + }, + "platform": { + "code": "ocw", + "name": "OCW" + }, + "resource_content_tags": [ + "Lecture Notes" + ], + "departments": [ + { + "department_id": "6", + "name": "Electrical Engineering and Computer Science" + } + ], + "certification": false, + "prices": [], + "runs": [ + { + "id": 2224, + "instructors": [ + { + "id": 966, + "first_name": "Gim", + "last_name": "Hom", + "full_name": "Gim Hom" + } + ], + "image": { + "id": 2288, + "url": "https://ocw.mit.edu/courses/6-091-hands-on-introduction-to-electrical-engineering-lab-skills-january-iap-2008/fc669b859c6991ac679eeb1a40ac69c5_6-091iap08.jpg", + "description": "In this lecture and lab course, students learn to use electrical instruments including function generators, oscilloscopes, and multimeters. Photograph courtesy of jbgeekdad on Flickr. (link over \"jbgeekdad\" to http://www.flickr.com/photos/jbgeekdad/1347629139/)", + "alt": "Photograph of a multimeter." + }, + "run_id": "8a0c893a1902034ecb6a319233ee0509", + "title": "Hands-On Introduction to Electrical Engineering Lab Skills", + "description": "This course introduces students to both passive and active electronic components (op-amps, 555 timers, TTL digital circuits). Basic analog and digital circuits and theory of operation are covered. The labs allow the students to master the use of electronic instruments and construct and/or solder several circuits. The labs also reinforce the concepts discussed in class with a hands-on approach and allow the students to gain significant experience with electrical instruments such as function generators, digital multimeters, oscilloscopes, logic analyzers and power supplies. In the last lab, the students build an electronic circuit that they can keep. The course is geared to freshmen and others who want an introduction to electronics circuits.\n\nThis course is offered during the Independent Activities Period (IAP), which is a special 4-week term at MIT that runs from the first week of January until the end of the month.\n", + "full_description": null, + "last_modified": "2023-12-06T14:18:49Z", + "published": true, + "languages": null, + "url": "https://ocw.mit.edu/courses/6-091-hands-on-introduction-to-electrical-engineering-lab-skills-january-iap-2008", + "level": "Undergraduate", + "slug": "courses/6-091-hands-on-introduction-to-electrical-engineering-lab-skills-january-iap-2008", + "availability": "Current", + "semester": "January IAP", + "year": 2008, + "start_date": null, + "end_date": null, + "enrollment_start": null, + "enrollment_end": null, + "prices": null, + "checksum": null + } + ], + "image": { + "id": 2288, + "url": "https://ocw.mit.edu/courses/6-091-hands-on-introduction-to-electrical-engineering-lab-skills-january-iap-2008/fc669b859c6991ac679eeb1a40ac69c5_6-091iap08.jpg", + "description": "In this lecture and lab course, students learn to use electrical instruments including function generators, oscilloscopes, and multimeters. Photograph courtesy of jbgeekdad on Flickr. (link over \"jbgeekdad\" to http://www.flickr.com/photos/jbgeekdad/1347629139/)", + "alt": "Photograph of a multimeter." + }, + "learning_path_parents": [], + "user_list_parents": [], + "resource_type": "course", + "course": { + "course_numbers": [ + { + "value": "6.091", + "department": { + "department_id": "6", + "name": "Electrical Engineering and Computer Science" + }, + "listing_type": "primary" + } + ] + }, + "readable_id": "6.091+january-iap_2008", + "title": "Hands-On Introduction to Electrical Engineering Lab Skills", + "description": "This course introduces students to both passive and active electronic components (op-amps, 555 timers, TTL digital circuits). Basic analog and digital circuits and theory of operation are covered. The labs allow the students to master the use of electronic instruments and construct and/or solder several circuits. The labs also reinforce the concepts discussed in class with a hands-on approach and allow the students to gain significant experience with electrical instruments such as function generators, digital multimeters, oscilloscopes, logic analyzers and power supplies. In the last lab, the students build an electronic circuit that they can keep. The course is geared to freshmen and others who want an introduction to electronics circuits.\n\nThis course is offered during the Independent Activities Period (IAP), which is a special 4-week term at MIT that runs from the first week of January until the end of the month.\n", + "full_description": null, + "last_modified": "2023-12-06T14:18:49Z", + "published": true, + "languages": null, + "url": "https://ocw.mit.edu/courses/6-091-hands-on-introduction-to-electrical-engineering-lab-skills-january-iap-2008", + "professional": false + }, + { + "id": 4097, + "topics": [ + { + "id": 32, + "name": "Engineering" + }, + { + "id": 1, + "name": "Science" + }, + { + "id": 41, + "name": "Chemistry" + }, + { + "id": 73, + "name": "Thermodynamics" + }, + { + "id": 140, + "name": "Materials Science and Engineering" + }, + { + "id": 209, + "name": "Physical Chemistry" + }, + { + "id": 30, + "name": "Physics" + } + ], + "offered_by": { + "code": "ocw", + "name": "OCW" + }, + "platform": { + "code": "ocw", + "name": "OCW" + }, + "resource_content_tags": [ + "Lecture Notes", + "Problem Sets with Solutions", + "Exams with Solutions" + ], + "departments": [ + { + "department_id": "3", + "name": "Materials Science and Engineering" + } + ], + "certification": false, + "prices": [], + "runs": [ + { + "id": 2737, + "instructors": [ + { + "id": 1459, + "first_name": "Samuel", + "last_name": "Allen", + "full_name": "Prof. Samuel Allen" + }, + { + "id": 744, + "first_name": "Thomas", + "last_name": "Eagar", + "full_name": "Prof. Thomas Eagar" + } + ], + "image": { + "id": 2800, + "url": "https://ocw.mit.edu/courses/3-205-thermodynamics-and-kinetics-of-materials-fall-2006/e36a3cb8c1066c086fd68559ce31ba3b_3-205f06.jpg", + "description": "Randomly dispersed foreign particles produce \"dizzy dendrites,\" in which the tree-like branches in the crystals tend to curve and split, instead of forming the straight, symmetric patterns typical of pure crystals. This sample was grown in an 80 nanometer-thick film of two blended polymers with randomly dispersed clay particles. (Photo courtesy of the National Institute of Standards and Technology (NIST).)", + "alt": "Photo of dizzy dendrite." + }, + "run_id": "cf9250f842fe6dc89189ad76c296a18f", + "title": "Thermodynamics and Kinetics of Materials", + "description": "This course explores materials and materials processes from the perspective of thermodynamics and kinetics. The thermodynamics aspect includes laws of thermodynamics, solution theory and equilibrium diagrams. The kinetics aspect includes diffusion, phase transformations, and the development of microstructure.\n", + "full_description": null, + "last_modified": "2023-12-06T15:25:12Z", + "published": true, + "languages": null, + "url": "https://ocw.mit.edu/courses/3-205-thermodynamics-and-kinetics-of-materials-fall-2006", + "level": "Graduate", + "slug": "courses/3-205-thermodynamics-and-kinetics-of-materials-fall-2006", + "availability": "Current", + "semester": "Fall", + "year": 2006, + "start_date": null, + "end_date": null, + "enrollment_start": null, + "enrollment_end": null, + "prices": null, + "checksum": null + } + ], + "image": { + "id": 2800, + "url": "https://ocw.mit.edu/courses/3-205-thermodynamics-and-kinetics-of-materials-fall-2006/e36a3cb8c1066c086fd68559ce31ba3b_3-205f06.jpg", + "description": "Randomly dispersed foreign particles produce \"dizzy dendrites,\" in which the tree-like branches in the crystals tend to curve and split, instead of forming the straight, symmetric patterns typical of pure crystals. This sample was grown in an 80 nanometer-thick film of two blended polymers with randomly dispersed clay particles. (Photo courtesy of the National Institute of Standards and Technology (NIST).)", + "alt": "Photo of dizzy dendrite." + }, + "learning_path_parents": [], + "user_list_parents": [], + "resource_type": "course", + "course": { + "course_numbers": [ + { + "value": "3.205", + "department": { + "department_id": "3", + "name": "Materials Science and Engineering" + }, + "listing_type": "primary" + } + ] + }, + "readable_id": "3.205+fall_2006", + "title": "Thermodynamics and Kinetics of Materials", + "description": "This course explores materials and materials processes from the perspective of thermodynamics and kinetics. The thermodynamics aspect includes laws of thermodynamics, solution theory and equilibrium diagrams. The kinetics aspect includes diffusion, phase transformations, and the development of microstructure.\n", + "full_description": null, + "last_modified": "2023-12-06T15:25:12Z", + "published": true, + "languages": null, + "url": "https://ocw.mit.edu/courses/3-205-thermodynamics-and-kinetics-of-materials-fall-2006", + "professional": false + }, + { + "id": 4238, + "topics": [ + { + "id": 4, + "name": "Social Science" + }, + { + "id": 38, + "name": "Humanities" + }, + { + "id": 11, + "name": "Communication" + }, + { + "id": 47, + "name": "Language" + }, + { + "id": 80, + "name": "Anthropology" + }, + { + "id": 79, + "name": "Cultural Anthropology" + }, + { + "id": 307, + "name": "English as a Second Language" + } + ], + "offered_by": { + "code": "ocw", + "name": "OCW" + }, + "platform": { + "code": "ocw", + "name": "OCW" + }, + "resource_content_tags": [ + "Written Assignments" + ], + "departments": [ + { + "department_id": "21G", + "name": "Global Studies and Languages" + } + ], + "certification": false, + "prices": [], + "runs": [ + { + "id": 2878, + "instructors": [ + { + "id": 1590, + "first_name": "Jane", + "last_name": "Dunphy", + "full_name": "Jane Dunphy" + } + ], + "image": { + "id": 2940, + "url": "https://ocw.mit.edu/courses/21g-221-communicating-in-american-culture-s-spring-2019/4d24a55ffb6894fe96be4df86c3ace02_21g-221s19.jpg", + "description": "", + "alt": "Barack Obama holding a microphone and speaking outdoors in front of a number of people." + }, + "run_id": "29859e86452391cf4b62bf2729ddc45c", + "title": "Communicating in American Culture(s)", + "description": "In _Communicating in American Culture(s)_, bilingual students examine how various aspects of American culture—history, geography, institutions, traditions, values—have shaped dominant Anglo-American communication norms and responses to critical events in the world. In addition, you can expect to practice and strengthen your analytical and communication skills in a carefully scaffolded manner, starting with frequent short writing and speaking tasks and progressing to longer, more formal tasks.\n", + "full_description": null, + "last_modified": "2023-12-06T15:53:03Z", + "published": true, + "languages": null, + "url": "https://ocw.mit.edu/courses/21g-221-communicating-in-american-culture-s-spring-2019", + "level": "Undergraduate", + "slug": "courses/21g-221-communicating-in-american-culture-s-spring-2019", + "availability": "Current", + "semester": "Spring", + "year": 2019, + "start_date": null, + "end_date": null, + "enrollment_start": null, + "enrollment_end": null, + "prices": null, + "checksum": null + } + ], + "image": { + "id": 2940, + "url": "https://ocw.mit.edu/courses/21g-221-communicating-in-american-culture-s-spring-2019/4d24a55ffb6894fe96be4df86c3ace02_21g-221s19.jpg", + "description": "", + "alt": "Barack Obama holding a microphone and speaking outdoors in front of a number of people." + }, + "learning_path_parents": [], + "user_list_parents": [], + "resource_type": "course", + "course": { + "course_numbers": [ + { + "value": "21G.221", + "department": { + "department_id": "21G", + "name": "Global Studies and Languages" + }, + "listing_type": "primary" + } + ] + }, + "readable_id": "21G.221+spring_2019", + "title": "Communicating in American Culture(s)", + "description": "In _Communicating in American Culture(s)_, bilingual students examine how various aspects of American culture—history, geography, institutions, traditions, values—have shaped dominant Anglo-American communication norms and responses to critical events in the world. In addition, you can expect to practice and strengthen your analytical and communication skills in a carefully scaffolded manner, starting with frequent short writing and speaking tasks and progressing to longer, more formal tasks.\n", + "full_description": null, + "last_modified": "2023-12-06T15:53:03Z", + "published": true, + "languages": null, + "url": "https://ocw.mit.edu/courses/21g-221-communicating-in-american-culture-s-spring-2019", + "professional": false + }, + { + "id": 5589, + "topics": [ + { + "id": 128, + "name": "Biochemistry" + }, + { + "id": 1, + "name": "Science" + }, + { + "id": 2, + "name": "Biology" + }, + { + "id": 214, + "name": "Molecular Biology" + }, + { + "id": 215, + "name": "Cell Biology" + } + ], + "offered_by": { + "code": "ocw", + "name": "OCW" + }, + "platform": { + "code": "ocw", + "name": "OCW" + }, + "resource_content_tags": [ + "Written Assignments", + "Presentation Assignments" + ], + "departments": [ + { + "department_id": "7", + "name": "Biology" + } + ], + "certification": false, + "prices": [], + "runs": [ + { + "id": 4229, + "instructors": [ + { + "id": 2473, + "first_name": "Megan", + "last_name": "Kizer", + "full_name": "Dr. Megan Kizer" + }, + { + "id": 2474, + "first_name": "Robbie", + "last_name": "Wilson", + "full_name": "Dr. Robbie Wilson" + } + ], + "image": { + "id": 4280, + "url": "https://ocw.mit.edu/courses/7-341-turning-evolutionary-dials-directed-evolution-techniques-for-climate-change-and-beyond-spring-2022/7-341s22.jpg", + "description": null, + "alt": "A multi-colored 3 dimensional illustration, shaped like a bumpy donut, representing the structure of an emzyme and its interconnected domains." + }, + "run_id": "360461e5a0bd4e26bad6f2a3eeefd668", + "title": "Turning Evolutionary Dials: Directed Evolution Techniques for Climate Change and Beyond", + "description": "This course will cover the many ways in which we have realized evolution in the laboratory toward functional biomolecules, such as protein and nucleic-acid-based therapeutics, enzymes that catalyze production of synthetic drugs, and carbon-dioxide capture molecules to lessen the impact of climate change. Students will both become familiar with the field of directed molecular evolution and learn how to critically analyze primary research papers, design research experiments, and present data relating to molecular biology and evolution. The importance of directed evolution in biomedical and biotechnological careers, both academic and industrial, will be highlighted.\n\nThis course is one of many [Advanced Undergraduate Seminars](https://biology.mit.edu/undergraduate/current-students/subject-offerings/advanced-undergraduate-seminars/) offered by the Biology Department at MIT. These seminars are tailored for students with an interest in using primary research literature to discuss and learn about current biological research in a highly interactive setting. Many instructors of the Advanced Undergraduate Seminars are postdoctoral scientists with a strong interest in teaching.", + "full_description": null, + "last_modified": "2023-12-06T14:28:06Z", + "published": true, + "languages": null, + "url": "https://ocw.mit.edu/courses/7-341-turning-evolutionary-dials-directed-evolution-techniques-for-climate-change-and-beyond-spring-2022", + "level": "Undergraduate", + "slug": "courses/7-341-turning-evolutionary-dials-directed-evolution-techniques-for-climate-change-and-beyond-spring-2022", + "availability": "Current", + "semester": "Spring", + "year": 2022, + "start_date": null, + "end_date": null, + "enrollment_start": null, + "enrollment_end": null, + "prices": null, + "checksum": null + } + ], + "image": { + "id": 4280, + "url": "https://ocw.mit.edu/courses/7-341-turning-evolutionary-dials-directed-evolution-techniques-for-climate-change-and-beyond-spring-2022/7-341s22.jpg", + "description": null, + "alt": "A multi-colored 3 dimensional illustration, shaped like a bumpy donut, representing the structure of an emzyme and its interconnected domains." + }, + "learning_path_parents": [], + "user_list_parents": [], + "resource_type": "course", + "course": { + "course_numbers": [ + { + "value": "7.341", + "department": { + "department_id": "7", + "name": "Biology" + }, + "listing_type": "primary" + } + ] + }, + "readable_id": "7.341+spring_2022", + "title": "Turning Evolutionary Dials: Directed Evolution Techniques for Climate Change and Beyond", + "description": "This course will cover the many ways in which we have realized evolution in the laboratory toward functional biomolecules, such as protein and nucleic-acid-based therapeutics, enzymes that catalyze production of synthetic drugs, and carbon-dioxide capture molecules to lessen the impact of climate change. Students will both become familiar with the field of directed molecular evolution and learn how to critically analyze primary research papers, design research experiments, and present data relating to molecular biology and evolution. The importance of directed evolution in biomedical and biotechnological careers, both academic and industrial, will be highlighted.\n\nThis course is one of many [Advanced Undergraduate Seminars](https://biology.mit.edu/undergraduate/current-students/subject-offerings/advanced-undergraduate-seminars/) offered by the Biology Department at MIT. These seminars are tailored for students with an interest in using primary research literature to discuss and learn about current biological research in a highly interactive setting. Many instructors of the Advanced Undergraduate Seminars are postdoctoral scientists with a strong interest in teaching.", + "full_description": null, + "last_modified": "2023-12-06T14:28:06Z", + "published": true, + "languages": null, + "url": "https://ocw.mit.edu/courses/7-341-turning-evolutionary-dials-directed-evolution-techniques-for-climate-change-and-beyond-spring-2022", + "professional": false + }, + { + "id": 4113, + "topics": [ + { + "id": 32, + "name": "Engineering" + }, + { + "id": 37, + "name": "Mathematics" + }, + { + "id": 36, + "name": "Probability and Statistics" + }, + { + "id": 29, + "name": "Systems Engineering" + } + ], + "offered_by": { + "code": "ocw", + "name": "OCW" + }, + "platform": { + "code": "ocw", + "name": "OCW" + }, + "resource_content_tags": [ + "Lecture Notes", + "Problem Sets with Solutions", + "Exams with Solutions" + ], + "departments": [ + { + "department_id": "22", + "name": "Nuclear Science and Engineering" + } + ], + "certification": false, + "prices": [], + "runs": [ + { + "id": 2753, + "instructors": [ + { + "id": 1445, + "first_name": "Michael", + "last_name": "Golay", + "full_name": "Prof. Michael Golay" + } + ], + "image": { + "id": 2816, + "url": "https://ocw.mit.edu/courses/22-38-probability-and-its-applications-to-reliability-quality-control-and-risk-assessment-fall-2005/6fbd15d7ca09fae7e66c97bd1a59c848_22-38f05.jpg", + "description": "A fault tree from the NASA Accident Analysis Team in Report of the Presidential Commission on the Space Shuttle Challenger Accident. (Courtesy of NASA.)", + "alt": "Fault tree for external tank and solid rocket booster." + }, + "run_id": "010c552ac85a7e0e278ca154294c3892", + "title": "Probability And Its Applications To Reliability, Quality Control, And Risk Assessment", + "description": "This course covers interpretations of the concept of probability. Topics include basic probability rules; random variables and distribution functions; functions of random variables; and applications to quality control and the reliability assessment of mechanical/electrical components, as well as simple structures and redundant systems. The course also considers elements of statistics; Bayesian methods in engineering; methods for reliability and risk assessment of complex systems (event-tree and fault-tree analysis, common-cause failures, human reliability models); uncertainty propagation in complex systems (Monte Carlo methods, Latin Hypercube Sampling); and an introduction to Markov models. Examples and applications are drawn from nuclear and other industries, waste repositories, and mechanical systems.\n", + "full_description": null, + "last_modified": "2023-12-06T14:33:16Z", + "published": true, + "languages": null, + "url": "https://ocw.mit.edu/courses/22-38-probability-and-its-applications-to-reliability-quality-control-and-risk-assessment-fall-2005", + "level": "Graduate", + "slug": "courses/22-38-probability-and-its-applications-to-reliability-quality-control-and-risk-assessment-fall-2005", + "availability": "Current", + "semester": "Fall", + "year": 2005, + "start_date": null, + "end_date": null, + "enrollment_start": null, + "enrollment_end": null, + "prices": null, + "checksum": null + } + ], + "image": { + "id": 2816, + "url": "https://ocw.mit.edu/courses/22-38-probability-and-its-applications-to-reliability-quality-control-and-risk-assessment-fall-2005/6fbd15d7ca09fae7e66c97bd1a59c848_22-38f05.jpg", + "description": "A fault tree from the NASA Accident Analysis Team in Report of the Presidential Commission on the Space Shuttle Challenger Accident. (Courtesy of NASA.)", + "alt": "Fault tree for external tank and solid rocket booster." + }, + "learning_path_parents": [], + "user_list_parents": [], + "resource_type": "course", + "course": { + "course_numbers": [ + { + "value": "22.38", + "department": { + "department_id": "22", + "name": "Nuclear Science and Engineering" + }, + "listing_type": "primary" + } + ] + }, + "readable_id": "22.38+fall_2005", + "title": "Probability And Its Applications To Reliability, Quality Control, And Risk Assessment", + "description": "This course covers interpretations of the concept of probability. Topics include basic probability rules; random variables and distribution functions; functions of random variables; and applications to quality control and the reliability assessment of mechanical/electrical components, as well as simple structures and redundant systems. The course also considers elements of statistics; Bayesian methods in engineering; methods for reliability and risk assessment of complex systems (event-tree and fault-tree analysis, common-cause failures, human reliability models); uncertainty propagation in complex systems (Monte Carlo methods, Latin Hypercube Sampling); and an introduction to Markov models. Examples and applications are drawn from nuclear and other industries, waste repositories, and mechanical systems.\n", + "full_description": null, + "last_modified": "2023-12-06T14:33:16Z", + "published": true, + "languages": null, + "url": "https://ocw.mit.edu/courses/22-38-probability-and-its-applications-to-reliability-quality-control-and-risk-assessment-fall-2005", + "professional": false + }, + { + "id": 4041, + "topics": [ + { + "id": 186, + "name": "Industrial Relations and Human Resource Management" + }, + { + "id": 7, + "name": "Business" + } + ], + "offered_by": { + "code": "ocw", + "name": "OCW" + }, + "platform": { + "code": "ocw", + "name": "OCW" + }, + "resource_content_tags": [ + "Lecture Notes", + "Exams" + ], + "departments": [ + { + "department_id": "15", + "name": "Sloan School of Management" + } + ], + "certification": false, + "prices": [], + "runs": [ + { + "id": 2681, + "instructors": [ + { + "id": 779, + "first_name": "Diane", + "last_name": "Burton", + "full_name": "Prof. Diane Burton" + }, + { + "id": 703, + "first_name": "Paul", + "last_name": "Osterman", + "full_name": "Prof. Paul Osterman" + } + ], + "image": { + "id": 2744, + "url": "https://ocw.mit.edu/courses/15-660-strategic-hr-management-spring-2003/10124a7188f51cee59a8bd9b3a0aad44_15-660s03.jpg", + "description": "", + "alt": "Bobsled team competing in Winter Olympics." + }, + "run_id": "d0d1f5f5bb6f3a2b8a13dd87abc76b8c", + "title": "Strategic HR Management", + "description": "This course is about both the design and execution of human resource management strategies. This course has two central themes: (1) How to think systematically and strategically about aspects of managing the organization's human assets, and (2) What really needs to be done to implement these policies and to achieve competitive advantage. It adopts the perspective of a general manager and addresses human resource topics (including reward systems, performance management, high-performance human resource systems, training and development, recruitment, retention, equal employment opportunity laws, work-force diversity, and union-management relationships) from a strategic perspective.\n", + "full_description": null, + "last_modified": "2023-12-06T14:18:56Z", + "published": true, + "languages": null, + "url": "https://ocw.mit.edu/courses/15-660-strategic-hr-management-spring-2003", + "level": "Graduate", + "slug": "courses/15-660-strategic-hr-management-spring-2003", + "availability": "Current", + "semester": "Spring", + "year": 2003, + "start_date": null, + "end_date": null, + "enrollment_start": null, + "enrollment_end": null, + "prices": null, + "checksum": null + } + ], + "image": { + "id": 2744, + "url": "https://ocw.mit.edu/courses/15-660-strategic-hr-management-spring-2003/10124a7188f51cee59a8bd9b3a0aad44_15-660s03.jpg", + "description": "", + "alt": "Bobsled team competing in Winter Olympics." + }, + "learning_path_parents": [], + "user_list_parents": [], + "resource_type": "course", + "course": { + "course_numbers": [ + { + "value": "15.660", + "department": { + "department_id": "15", + "name": "Sloan School of Management" + }, + "listing_type": "primary" + } + ] + }, + "readable_id": "15.660+spring_2003", + "title": "Strategic HR Management", + "description": "This course is about both the design and execution of human resource management strategies. This course has two central themes: (1) How to think systematically and strategically about aspects of managing the organization's human assets, and (2) What really needs to be done to implement these policies and to achieve competitive advantage. It adopts the perspective of a general manager and addresses human resource topics (including reward systems, performance management, high-performance human resource systems, training and development, recruitment, retention, equal employment opportunity laws, work-force diversity, and union-management relationships) from a strategic perspective.\n", + "full_description": null, + "last_modified": "2023-12-06T14:18:56Z", + "published": true, + "languages": null, + "url": "https://ocw.mit.edu/courses/15-660-strategic-hr-management-spring-2003", + "professional": false + }, + { + "id": 4698, + "topics": [ + { + "id": 156, + "name": "Linguistics" + }, + { + "id": 182, + "name": "Phonology" + }, + { + "id": 38, + "name": "Humanities" + } + ], + "offered_by": { + "code": "ocw", + "name": "OCW" + }, + "platform": { + "code": "ocw", + "name": "OCW" + }, + "resource_content_tags": [ + "Lecture Notes", + "Written Assignments", + "Problem Sets" + ], + "departments": [ + { + "department_id": "24", + "name": "Linguistics and Philosophy" + } + ], + "certification": false, + "prices": [], + "runs": [ + { + "id": 3338, + "instructors": [ + { + "id": 699, + "first_name": "Edward", + "last_name": "Flemming", + "full_name": "Prof. Edward Flemming" + } + ], + "image": { + "id": 3398, + "url": "https://ocw.mit.edu/courses/24-915-linguistic-phonetics-fall-2015/ceb01a247960c746f44324ee761b9cdd_24-915f15.jpg", + "description": "CHP image of a waveform for the f15 iteration of 24.915.", + "alt": "On the right is a waveform in black on a blue background, and on the left shows trees reflecting on a still lake with snowy mountains in the background." + }, + "run_id": "9bff24096453777f38e55c662d0e7767", + "title": "Linguistic Phonetics", + "description": "This course is about the study of speech sounds; how we produce and perceive them and their acoustic properties. Topics include the influence of the production and perception systems on phonological patterns and sound change, students learn acoustic analysis and experimental techniques. Students taking the graduate version complete different assignments.\n", + "full_description": null, + "last_modified": "2023-12-06T15:32:05Z", + "published": true, + "languages": null, + "url": "https://ocw.mit.edu/courses/24-915-linguistic-phonetics-fall-2015", + "level": "Undergraduate, Graduate", + "slug": "courses/24-915-linguistic-phonetics-fall-2015", + "availability": "Current", + "semester": "Fall", + "year": 2015, + "start_date": null, + "end_date": null, + "enrollment_start": null, + "enrollment_end": null, + "prices": null, + "checksum": null + } + ], + "image": { + "id": 3398, + "url": "https://ocw.mit.edu/courses/24-915-linguistic-phonetics-fall-2015/ceb01a247960c746f44324ee761b9cdd_24-915f15.jpg", + "description": "CHP image of a waveform for the f15 iteration of 24.915.", + "alt": "On the right is a waveform in black on a blue background, and on the left shows trees reflecting on a still lake with snowy mountains in the background." + }, + "learning_path_parents": [], + "user_list_parents": [], + "resource_type": "course", + "course": { + "course_numbers": [ + { + "value": "24.915", + "department": { + "department_id": "24", + "name": "Linguistics and Philosophy" + }, + "listing_type": "primary" + }, + { + "value": "24.963", + "department": { + "department_id": "24", + "name": "Linguistics and Philosophy" + }, + "listing_type": "cross-listed" + } + ] + }, + "readable_id": "24.915+fall_2015", + "title": "Linguistic Phonetics", + "description": "This course is about the study of speech sounds; how we produce and perceive them and their acoustic properties. Topics include the influence of the production and perception systems on phonological patterns and sound change, students learn acoustic analysis and experimental techniques. Students taking the graduate version complete different assignments.\n", + "full_description": null, + "last_modified": "2023-12-06T15:32:05Z", + "published": true, + "languages": null, + "url": "https://ocw.mit.edu/courses/24-915-linguistic-phonetics-fall-2015", + "professional": false + }, + { + "id": 4541, + "topics": [ + { + "id": 4, + "name": "Social Science" + }, + { + "id": 7, + "name": "Business" + }, + { + "id": 8, + "name": "Political Science" + }, + { + "id": 9, + "name": "Society" + }, + { + "id": 235, + "name": "Women's Studies" + }, + { + "id": 15, + "name": "Leadership" + }, + { + "id": 208, + "name": "Gender Studies" + }, + { + "id": 49, + "name": "Public Policy" + }, + { + "id": 308, + "name": "Social Welfare" + }, + { + "id": 53, + "name": "Public Administration" + }, + { + "id": 281, + "name": "American Politics" + } + ], + "offered_by": { + "code": "ocw", + "name": "OCW" + }, + "platform": { + "code": "ocw", + "name": "OCW" + }, + "resource_content_tags": [ + "Written Assignments with Examples", + "Instructor Insights" + ], + "departments": [ + { + "department_id": "WGS", + "name": "Women's and Gender Studies" + } + ], + "certification": false, + "prices": [], + "runs": [ + { + "id": 3181, + "instructors": [ + { + "id": 1848, + "first_name": "Mindy", + "last_name": "Fried", + "full_name": "Dr. Mindy Fried" + } + ], + "image": { + "id": 3241, + "url": "https://ocw.mit.edu/courses/wgs-s10-gender-power-leadership-and-the-workplace-spring-2014/2888e7c93814bc3ade6901fbee38bb10_wgs-s10s14.jpg", + "description": "During an International Women's Day demonstration in Valencia, Spain, women gathered to collectively breastfeed to advocate for their rights as working mothers. Image courtesy of Amadeu Sanz on Flickr. CC NC-BY-SA.", + "alt": "A photograph of a group of women sit on the ground nursing their babies." + }, + "run_id": "18f30b4ebe6ae04244d9b6fdf7de4e9e", + "title": "Gender, Power, Leadership, and the Workplace", + "description": "This course will provide students with an analytic framework to understand the roles that gender, race, and class play in defining and determining access to leadership and power in the U.S., especially in the context of the workplace. We will explore women and men in leadership positions within the corporate, political and non-profit sectors, with attention to the roles of women of color and immigrant women within this context. We will also look at specific policies such as affirmative action, parental leave, child-care policy, and working-time policies and the role they play–or could play–in achieving parity. We will further investigate ways in which these policies address gender, racial, and class inequities, and think critically about mechanisms for change. The course will be highly interactive, and will combine texts, theater, videos and visual arts.\n", + "full_description": null, + "last_modified": "2023-12-06T14:14:29Z", + "published": true, + "languages": null, + "url": "https://ocw.mit.edu/courses/wgs-s10-gender-power-leadership-and-the-workplace-spring-2014", + "level": "Undergraduate", + "slug": "courses/wgs-s10-gender-power-leadership-and-the-workplace-spring-2014", + "availability": "Current", + "semester": "Spring", + "year": 2014, + "start_date": null, + "end_date": null, + "enrollment_start": null, + "enrollment_end": null, + "prices": null, + "checksum": null + } + ], + "image": { + "id": 3241, + "url": "https://ocw.mit.edu/courses/wgs-s10-gender-power-leadership-and-the-workplace-spring-2014/2888e7c93814bc3ade6901fbee38bb10_wgs-s10s14.jpg", + "description": "During an International Women's Day demonstration in Valencia, Spain, women gathered to collectively breastfeed to advocate for their rights as working mothers. Image courtesy of Amadeu Sanz on Flickr. CC NC-BY-SA.", + "alt": "A photograph of a group of women sit on the ground nursing their babies." + }, + "learning_path_parents": [], + "user_list_parents": [], + "resource_type": "course", + "course": { + "course_numbers": [ + { + "value": "WGS.S10", + "department": { + "department_id": "WGS", + "name": "Women's and Gender Studies" + }, + "listing_type": "primary" + } + ] + }, + "readable_id": "WGS.S10+spring_2014", + "title": "Gender, Power, Leadership, and the Workplace", + "description": "This course will provide students with an analytic framework to understand the roles that gender, race, and class play in defining and determining access to leadership and power in the U.S., especially in the context of the workplace. We will explore women and men in leadership positions within the corporate, political and non-profit sectors, with attention to the roles of women of color and immigrant women within this context. We will also look at specific policies such as affirmative action, parental leave, child-care policy, and working-time policies and the role they play–or could play–in achieving parity. We will further investigate ways in which these policies address gender, racial, and class inequities, and think critically about mechanisms for change. The course will be highly interactive, and will combine texts, theater, videos and visual arts.\n", + "full_description": null, + "last_modified": "2023-12-06T14:14:29Z", + "published": true, + "languages": null, + "url": "https://ocw.mit.edu/courses/wgs-s10-gender-power-leadership-and-the-workplace-spring-2014", + "professional": false + }, + { + "id": 5246, + "topics": [ + { + "id": 36, + "name": "Probability and Statistics" + }, + { + "id": 37, + "name": "Mathematics" + }, + { + "id": 159, + "name": "Mathematical Analysis" + } + ], + "offered_by": { + "code": "ocw", + "name": "OCW" + }, + "platform": { + "code": "ocw", + "name": "OCW" + }, + "resource_content_tags": [ + "Lecture Notes", + "Problem Sets" + ], + "departments": [ + { + "department_id": "18", + "name": "Mathematics" + } + ], + "certification": false, + "prices": [], + "runs": [ + { + "id": 3886, + "instructors": [ + { + "id": 1175, + "first_name": "Scott", + "last_name": "Sheffield", + "full_name": "Prof. Scott Sheffield" + } + ], + "image": { + "id": 3942, + "url": "https://ocw.mit.edu/courses/18-175-theory-of-probability-spring-2014/27c7527c8726658ad43f1e098d73b67e_18-175s14.jpg", + "description": "Diffusion through a liquid. Image courtesy of Sanath Kumar on Flickr. CC BY 2.0. ", + "alt": "Image of a blue substance diffusing through a clear liquid." + }, + "run_id": "3ffc15bd89dc2b022008638010618126", + "title": "Theory of Probability", + "description": "This course covers topics such as sums of independent random variables, central limit phenomena, infinitely divisible laws, Levy processes, Brownian motion, conditioning, and martingales.\n", + "full_description": null, + "last_modified": "2023-12-06T15:05:29Z", + "published": true, + "languages": null, + "url": "https://ocw.mit.edu/courses/18-175-theory-of-probability-spring-2014", + "level": "Graduate", + "slug": "courses/18-175-theory-of-probability-spring-2014", + "availability": "Current", + "semester": "Spring", + "year": 2014, + "start_date": null, + "end_date": null, + "enrollment_start": null, + "enrollment_end": null, + "prices": null, + "checksum": null + } + ], + "image": { + "id": 3942, + "url": "https://ocw.mit.edu/courses/18-175-theory-of-probability-spring-2014/27c7527c8726658ad43f1e098d73b67e_18-175s14.jpg", + "description": "Diffusion through a liquid. Image courtesy of Sanath Kumar on Flickr. CC BY 2.0. ", + "alt": "Image of a blue substance diffusing through a clear liquid." + }, + "learning_path_parents": [], + "user_list_parents": [], + "resource_type": "course", + "course": { + "course_numbers": [ + { + "value": "18.175", + "department": { + "department_id": "18", + "name": "Mathematics" + }, + "listing_type": "primary" + } + ] + }, + "readable_id": "18.175+spring_2014", + "title": "Theory of Probability", + "description": "This course covers topics such as sums of independent random variables, central limit phenomena, infinitely divisible laws, Levy processes, Brownian motion, conditioning, and martingales.\n", + "full_description": null, + "last_modified": "2023-12-06T15:05:29Z", + "published": true, + "languages": null, + "url": "https://ocw.mit.edu/courses/18-175-theory-of-probability-spring-2014", + "professional": false + }, + { + "id": 4726, + "topics": [ + { + "id": 17, + "name": "Marketing" + }, + { + "id": 10, + "name": "Entrepreneurship" + }, + { + "id": 7, + "name": "Business" + } + ], + "offered_by": { + "code": "ocw", + "name": "OCW" + }, + "platform": { + "code": "ocw", + "name": "OCW" + }, + "resource_content_tags": [ + "Lecture Notes", + "Written Assignments with Examples" + ], + "departments": [ + { + "department_id": "15", + "name": "Sloan School of Management" + } + ], + "certification": false, + "prices": [], + "runs": [ + { + "id": 3366, + "instructors": [ + { + "id": 1976, + "first_name": "Jin", + "last_name": "Kim", + "full_name": "Prof. Jin Gyo Kim" + } + ], + "image": { + "id": 3426, + "url": "https://ocw.mit.edu/courses/15-835-entrepreneurial-marketing-spring-2002/6efc1711d37e8e136ae578131f10f11a_15-835s02.jpg", + "description": "Major themes of entrepreneurial marketing. (Diagram by Prof. Jin Gyo Kim.)", + "alt": "An illustration containing the central themes and questions of entrepreneurial ventures and marketing." + }, + "run_id": "9e4b93ac9f130565f1fb9a238751ab9b", + "title": "Entrepreneurial Marketing", + "description": "This course clarifies key marketing concepts, methods, and strategic issues relevant for start-up and early-stage entrepreneurs. At this course, there are two major questions:\n\n1. **Marketing Question:** What and how am I selling to whom?\n2. **New Venture Question:** How do I best leverage my limited marketing recourses?\n\nSpecifically, this course is designed to give students a broad and deep understanding of such topics as:\n\n1. What are major strategic constraints and issues confronted by entrepreneurs today?\n2. How can one identify and evaluate marketing opportunities?\n3. How do entrepreneurs achieve competitive advantages given limited marketing resources?\n4. What major marketing/sales tools are most useful in an entrepreneurial setting?\n\nBecause there is no universal marketing solution applicable to all entrepreneurial ventures, this course is designed to help students develop a flexible way of thinking about marketing problems in general.\n", + "full_description": null, + "last_modified": "2023-12-06T15:35:01Z", + "published": true, + "languages": null, + "url": "https://ocw.mit.edu/courses/15-835-entrepreneurial-marketing-spring-2002", + "level": "Graduate", + "slug": "courses/15-835-entrepreneurial-marketing-spring-2002", + "availability": "Current", + "semester": "Spring", + "year": 2002, + "start_date": null, + "end_date": null, + "enrollment_start": null, + "enrollment_end": null, + "prices": null, + "checksum": null + } + ], + "image": { + "id": 3426, + "url": "https://ocw.mit.edu/courses/15-835-entrepreneurial-marketing-spring-2002/6efc1711d37e8e136ae578131f10f11a_15-835s02.jpg", + "description": "Major themes of entrepreneurial marketing. (Diagram by Prof. Jin Gyo Kim.)", + "alt": "An illustration containing the central themes and questions of entrepreneurial ventures and marketing." + }, + "learning_path_parents": [], + "user_list_parents": [], + "resource_type": "course", + "course": { + "course_numbers": [ + { + "value": "15.835", + "department": { + "department_id": "15", + "name": "Sloan School of Management" + }, + "listing_type": "primary" + } + ] + }, + "readable_id": "15.835+spring_2002", + "title": "Entrepreneurial Marketing", + "description": "This course clarifies key marketing concepts, methods, and strategic issues relevant for start-up and early-stage entrepreneurs. At this course, there are two major questions:\n\n1. **Marketing Question:** What and how am I selling to whom?\n2. **New Venture Question:** How do I best leverage my limited marketing recourses?\n\nSpecifically, this course is designed to give students a broad and deep understanding of such topics as:\n\n1. What are major strategic constraints and issues confronted by entrepreneurs today?\n2. How can one identify and evaluate marketing opportunities?\n3. How do entrepreneurs achieve competitive advantages given limited marketing resources?\n4. What major marketing/sales tools are most useful in an entrepreneurial setting?\n\nBecause there is no universal marketing solution applicable to all entrepreneurial ventures, this course is designed to help students develop a flexible way of thinking about marketing problems in general.\n", + "full_description": null, + "last_modified": "2023-12-06T15:35:01Z", + "published": true, + "languages": null, + "url": "https://ocw.mit.edu/courses/15-835-entrepreneurial-marketing-spring-2002", + "professional": false + } +] diff --git a/tests/test_api_response.py b/tests/test_api_response.py new file mode 100644 index 0000000..86d416c --- /dev/null +++ b/tests/test_api_response.py @@ -0,0 +1,30 @@ +import unittest +from ocw_oer_export.constants import API_URL +from ocw_oer_export.client import make_request + + +class APITestCase(unittest.TestCase): + """Test suite for ensuring the MIT OpenCourseWare API response + contains all necessary fields for OER template.""" + + def test_api_fields(self): + """Test that we have all the needed fields in the API. + Fields required: title, url, description, topics, instructors, course_feature""" + page_size = 1 + response = make_request(API_URL, page_size) + + # Ensure the request was successful + self.assertEqual(response.status_code, 200) + + api_data = response.json().get("results", []) + + # Assert that only one item is returned + self.assertEqual(len(api_data), 1) + + item = api_data[0] + self.assertIn("title", item) + self.assertIn("url", item) + self.assertIn("description", item) + self.assertIn("topics", item) + self.assertIn("resource_content_tags", item) + self.assertIn("instructors", item["runs"][0]) diff --git a/tests/test_csv_creation_from_json.py b/tests/test_csv_creation_from_json.py new file mode 100644 index 0000000..b379bc3 --- /dev/null +++ b/tests/test_csv_creation_from_json.py @@ -0,0 +1,38 @@ +import os +import unittest +from ocw_oer_export.create_csv import create_csv +from ocw_oer_export.data_handler import extract_data_from_file +from ocw_oer_export.utilities import delete_file + + +class CSVCreationTestCase(unittest.TestCase): + """Test suite for verifying CSV creation from JSON data.""" + + @classmethod + def setUpClass(cls): + """Class setup that runs once before all tests.""" + cls.test_dir = os.path.dirname(__file__) + cls.expected_csv_path = os.path.join(cls.test_dir, "expected_courses.csv") + cls.sample_json_path = os.path.join(cls.test_dir, "sample_courses.json") + cls.generated_csv_path = os.path.join(cls.test_dir, "test_output.csv") + + def test_csv_creation_from_json(self): + """Test the CSV generated from JSON matches the expected CSV content.""" + create_csv( + source="json", + input_file=self.sample_json_path, + output_file=self.generated_csv_path, + ) + generated_csv_data = extract_data_from_file(self.generated_csv_path) + expected_csv_data = extract_data_from_file(self.expected_csv_path) + + self.assertEqual( + generated_csv_data, + expected_csv_data, + "The generated CSV file does not match the expected CSV content.", + ) + + @classmethod + def tearDownClass(cls): + """Class teardown that runs once after all tests.""" + delete_file(cls.generated_csv_path) diff --git a/tests/test_description_cleanup.py b/tests/test_description_cleanup.py new file mode 100644 index 0000000..cf76d40 --- /dev/null +++ b/tests/test_description_cleanup.py @@ -0,0 +1,44 @@ +import unittest +from ocw_oer_export.utilities import text_cleanup + + +class DescriptionCleanupTests(unittest.TestCase): + """Test suite for verifying the functionality of the description cleanup process.""" + + def test_markdown_bold_removal(self): + """Test that markdown bold syntax is removed.""" + sample_text = r"**2\. Adherence to guidelines (10% total grade):**" + cleaned_text = text_cleanup(sample_text) + self.assertEqual(cleaned_text, "2. Adherence to guidelines (10% total grade):") + + def test_markdown_header_removal(self): + """Test that markdown header syntax is removed.""" + sample_text = "### Other Notes" + cleaned_text = text_cleanup(sample_text) + self.assertEqual(cleaned_text, "Other Notes") + + def test_custom_markup_removal(self): + """Test that OCW Studio's custom markup is removed.""" + sample_text = ( + "{{< tableopen >}}{{< theadopen >}}{{< tropen >}}" + "{{< thopen >}}Points{{< thclose >}}{{< thopen >}}" + " Assessment{{< thclose >}}{{< trclose >}}{{< theadclose >}}" + "{{< tableclose >}}" + ) + cleaned_text = text_cleanup(sample_text) + self.assertEqual(cleaned_text, "Points Assessment") + + def test_newline_normalization(self): + """Test that extra newlines are removed and only single newlines remain.""" + + sample_text = "CHINESE COURSES\n\n\n\nCOURSE SITES\n\n\nChinese I (Fall 2014)" + cleaned_text = text_cleanup(sample_text) + expected_description = "CHINESE COURSES\nCOURSE SITES\nChinese I (Fall 2014)" + self.assertEqual(cleaned_text, expected_description) + + def test_empty_curly_brackets_removal(self): + """Test the removal of content within curly brackets and subsequent empty lines.""" + sample_text = "CHINESE COURSES\n{{}}\n{{}}\nCOURSE SITES" + cleaned_text = text_cleanup(sample_text) + expected_description = "CHINESE COURSES\nCOURSE SITES" + self.assertEqual(cleaned_text, expected_description)