diff --git a/csm_web/scheduler/factories.py b/csm_web/scheduler/factories.py index 38ebddf3..36dee9d4 100644 --- a/csm_web/scheduler/factories.py +++ b/csm_web/scheduler/factories.py @@ -1,4 +1,5 @@ import random +import time from datetime import timedelta import factory @@ -47,6 +48,31 @@ def evaluate_faker(faker_obj: factory.Faker): "Architecture", ) +COURSE_INFO = [ + # (name, title, is_restricted) + ("CSM61A", "Structure and Interpretation of Computer Programs", False), + ("CSM61B", "Data Structures", False), + ("CSM61C", "Machine Structures", False), + ("CSM70", "Discrete Mathematics and Probability Theory", False), + ("CSM88", "Computational Structures in Data Science", False), + ("CSM16A", "Designing Information Devices and Systems I", False), + ("CSM16B", "Designing Information Devices and Systems II", False), + ("CS61A", "Structure and Interpretation of Computer Programs", True), + ("CS61B", "Data Structures", True), + ("CS61C", "Machine Structures", True), + ("CS70", "Discrete Mathematics and Probability Theory", True), + ("CS88", "Computational Structures in Data Science", True), + ("EECS16A", "Designing Information Devices and Systems I", True), + ("EECS16B", "Designing Information Devices and Systems II", True), +] + +LARGE_COURSE_INFO = [ + # (name, title, (min_sections, max_sections), (min_students, max_students)) + ("LARGE1", "Course with many small sections", (40, 60), (3, 10)), + ("LARGE2", "Course with a few large sections", (5, 10), (40, 60)), + ("LARGE3", "Course with many large sections", (40, 60), (40, 60)), +] + class CourseFactory(factory.django.DjangoModelFactory): class Meta: @@ -267,6 +293,65 @@ def build_attendances_for(student): return attendance_objects +def build_spacetimes_occurrences_for(section, true_random_spacetimes=True): + """ + Build spacetimes and section occurrences for this section. + + Section should be associated with a course that has a specified + `section_start` and `valid_until` field. + """ + + spacetime_objects = [] + section_occurrence_objects = [] + + # spacetimes for section; manually create because factory._create is not called + num_spacetimes = random.choices([1, 2], [0.75, 0.25])[0] + if true_random_spacetimes: + # We want to ensure that if there are 2 spacetimes for a section + # that they are for different days of the week + spacetime_days = random.sample(DayOfWeekField.DAYS, num_spacetimes) + for day in spacetime_days: + spacetime_objects.append( + SpacetimeFactory.build(section=section, day_of_week=day) + ) + else: + # use realistic choices for spacetimes + if num_spacetimes == 1: + # no weekends + spacetime_days = [random.choice(DayOfWeekField.DAYS[:5])] + spacetime_objects.append( + SpacetimeFactory.build(section=section, day_of_week=spacetime_days[0]) + ) + else: + # choose one of M/W, Tu/Th, W/F + spacetime_days = random.choice( + [ + (DayOfWeekField.DAYS[0], DayOfWeekField.DAYS[2]), + (DayOfWeekField.DAYS[1], DayOfWeekField.DAYS[3]), + (DayOfWeekField.DAYS[2], DayOfWeekField.DAYS[4]), + ] + ) + for day in spacetime_days: + spacetime_objects.append( + SpacetimeFactory.build(section=section, day_of_week=day) + ) + + current_date = week_bounds(section.mentor.course.section_start)[0] + spacetime_day_numbers = [ + day_to_number(day_of_week) for day_of_week in spacetime_days + ] + # create section occurrences + while current_date < section.mentor.course.valid_until: + for spacetime_day in spacetime_day_numbers: + date = current_date + timedelta(days=spacetime_day) + section_occurrence_objects.append( + SectionOccurrenceFactory.build(section=section, date=date) + ) + current_date += timedelta(weeks=1) + + return spacetime_objects, section_occurrence_objects + + def demoify_user(user, username): """Make the user an admin.""" user.is_staff = True @@ -289,18 +374,28 @@ def create_demo_account(): - mentor for another course's section - coordinator for a third course - whitelist for three random restricted courses + - coordinator for all the LARGE* courses Also create a new demo mentor account associated with a new course that the first demo_user account is a coordindator for. This last account would primarily be used for matcher testing, as 50 other mentors (not associated with any section) are created for this course. + This demo mentor account will also be a mentor for a section in each of the LARGE* courses. """ + + course_filter = [info[0] for info in COURSE_INFO] + large_course_filter = [info[0] for info in LARGE_COURSE_INFO] + # get a random mentor with 2 spacetimes demo_mentor = random.choice( Mentor.objects.annotate(Count("section__spacetimes")).filter( - section__spacetimes__count=2, course__is_restricted=False + section__spacetimes__count=2, + course__is_restricted=False, + course__name__in=course_filter, ) ) + demo_user = demo_mentor.user + # associate it with a second section (take over another mentor) second_section = random.choice( Section.objects.filter( @@ -308,34 +403,36 @@ def create_demo_account(): ).exclude(pk=demo_mentor.section.pk) ) demo_mentor_2 = second_section.mentor - demo_mentor_2.user = demo_mentor.user + demo_mentor_2.user = demo_user demo_mentor_2.save() # make user admin and change username - demoify_user(demo_mentor.user, "demo_user") + demoify_user(demo_user, "demo_user") # associate user as student in another section (take over another student) demo_student = random.choice( - Student.objects.filter(course__is_restricted=True).exclude( - course=demo_mentor.course - ) + Student.objects.filter( + course__is_restricted=True, course__name__in=course_filter + ).exclude(course=demo_mentor.course) ) - demo_student.user = demo_mentor.user - demo_student.course.whitelist.add(demo_mentor.user) + demo_student.user = demo_user + demo_student.course.whitelist.add(demo_user) demo_student.save() # associate user as coordinator of another course (create a new coordinator) demo_coord = Coordinator.objects.create( user=demo_mentor.user, course=random.choice( - Course.objects.filter(is_restricted=False).exclude( + Course.objects.filter(is_restricted=False, name__in=course_filter).exclude( pk__in=(demo_mentor.course.pk, demo_student.course.pk) ) ), ) # whitelist to a random selection of restricted courses - restricted_courses = Course.objects.filter(is_restricted=True) + restricted_courses = Course.objects.filter( + is_restricted=True, name__in=course_filter + ) first_whitelist_course = random.choice(restricted_courses) second_whitelist_course = random.choice( restricted_courses.exclude(pk=first_whitelist_course.pk) @@ -346,19 +443,24 @@ def create_demo_account(): ) ) - first_whitelist_course.whitelist.add(demo_mentor.user) - second_whitelist_course.whitelist.add(demo_mentor.user) - coord_whitelist_course.whitelist.add(demo_mentor.user) + first_whitelist_course.whitelist.add(demo_user) + second_whitelist_course.whitelist.add(demo_user) + coord_whitelist_course.whitelist.add(demo_user) restricted_demo_coord = Coordinator.objects.create( - user=demo_mentor.user, course=coord_whitelist_course + user=demo_user, course=coord_whitelist_course ) - print( - """ + # associate user as coordinator to all of the large sections + for large_course_name in large_course_filter: + Coordinator.objects.create( + user=demo_user, course=Course.objects.get(name=large_course_name) + ) + + print(""" A demo account has been created with username 'demo_user' and password 'pass' Log in at localhost:8000/admin/ - """ - ) + """) + # make demo_user a coord for one more course coord_2_whitelist_course = random.choice( Course.objects.exclude( @@ -368,7 +470,7 @@ def create_demo_account(): demo_coord.course.pk, restricted_demo_coord.course.pk, ) - ) + ).filter(name__in=course_filter) ) demo_coord_2 = Coordinator.objects.create( user=demo_mentor.user, course=coord_2_whitelist_course @@ -379,25 +481,30 @@ def create_demo_account(): mentors = MentorFactory.create_batch(50, course=demo_coord_2.course) # allow one mentor to login through admin menu - demoify_user(mentors[0].user, "demo_mentor") - print( - """ + demo_mentor_user = mentors[0].user + demoify_user(demo_mentor_user, "demo_mentor") + + # associate the same user as a mentor for all the large courses (take over a mentor) + for large_course_name in large_course_filter: + large_course_section = random.choice( + Section.objects.filter(mentor__course__name=large_course_name) + ) + large_course_section.mentor.user = demo_mentor_user + + print(""" A demo mentor has been created with username 'demo_mentor' and password 'pass' Log in at localhost:8000/admin/ - """ - ) + """) def confirm_run(): """Display warning message for user to confirm flushing the database.""" - choice = input( - """You have requested a flush of the database. + choice = input("""You have requested a flush of the database. This will DELETE EVERYTHING IN THE DATABASE, and return all tables to an empty state. Are you sure you want to do this? - Type 'yes' to continue, or 'no' to abort: """ - ) + Type 'yes' to continue, or 'no' to abort: """) while choice not in ("yes", "no"): choice = input("Please type 'yes' or 'no' (without the quotes): ") return choice == "yes" @@ -412,25 +519,9 @@ def generate_test_data(preconfirm=False): return management.call_command("flush", interactive=False) - course_info = [ - # (name, title, is_restricted) - ("CSM61A", "Structure and Interpretation of Computer Programs", False), - ("CSM61B", "Data Structures", False), - ("CSM61C", "Machine Structures", False), - ("CSM70", "Discrete Mathematics and Probability Theory", False), - ("CSM88", "Computational Structures in Data Science", False), - ("CSM16A", "Designing Information Devices and Systems I", False), - ("CSM16B", "Designing Information Devices and Systems II", False), - ("CS61A", "Structure and Interpretation of Computer Programs", True), - ("CS61B", "Data Structures", True), - ("CS61C", "Machine Structures", True), - ("CS70", "Discrete Mathematics and Probability Theory", True), - ("CS88", "Computational Structures in Data Science", True), - ("EECS16A", "Designing Information Devices and Systems I", True), - ("EECS16B", "Designing Information Devices and Systems II", True), - ] - print("Generating test data...") + _start_time = time.perf_counter_ns() + enrollment_start = timezone.now() - timedelta(days=14) enrollment_end = timezone.now() + timedelta(days=50) valid_until = timezone.now() + timedelta(days=100) @@ -443,8 +534,9 @@ def generate_test_data(preconfirm=False): section_occurrence_objects = [] student_objects = [] - print("Creating model instances...") - for course_name, course_title, course_restricted in course_info: + print("Creating model instances... ", end="") + _create_models_start = time.perf_counter_ns() + for course_name, course_title, course_restricted in COURSE_INFO: course = CourseFactory.build( name=course_name, title=course_title, @@ -454,6 +546,8 @@ def generate_test_data(preconfirm=False): is_restricted=course_restricted, ) course_objects.append(course) + + # sections for this course for _ in range(random.randint(5, 10)): mentor_user = UserFactory.build() user_objects.append(mentor_user) @@ -462,29 +556,64 @@ def generate_test_data(preconfirm=False): section = SectionFactory.build(mentor=mentor) section_objects.append(section) - # spacetimes for section; manually create because facotyr._create is not called - num_spacetimes = random.randint(1, 2) - # We want to ensure that if there are 2 spacetimes for a section - # that they are for different days of the week - spacetime_days = random.sample(DayOfWeekField.DAYS, num_spacetimes) - for day in spacetime_days: - spacetime_objects.append( - SpacetimeFactory.build(section=section, day_of_week=day) - ) + # create spacetimes and section occurrences + cur_spacetimes, cur_section_occurrences = build_spacetimes_occurrences_for( + section, true_random_spacetimes=True + ) + spacetime_objects.extend(cur_spacetimes) + section_occurrence_objects.extend(cur_section_occurrences) - current_date = week_bounds(section.mentor.course.section_start)[0] - spacetime_day_numbers = [ - day_to_number(day_of_week) for day_of_week in spacetime_days - ] - while current_date < section.mentor.course.valid_until: - for spacetime_day in spacetime_day_numbers: - date = current_date + timedelta(days=spacetime_day) - section_occurrence_objects.append( - SectionOccurrenceFactory.build(section=section, date=date) + # create students for the section + student_users = UserFactory.build_batch(random.randint(0, section.capacity)) + user_objects.extend(student_users) + students = [] + for student_user in student_users: + students.append( + StudentFactory.build( + section=section, course=course, user=student_user ) - current_date += timedelta(weeks=1) + ) + student_objects.extend(students) - student_users = UserFactory.build_batch(random.randint(0, section.capacity)) + # courses with many sections/students + for ( + course_name, + course_title, + (min_sections, max_sections), + (min_students, max_students), + ) in LARGE_COURSE_INFO: + course = CourseFactory.build( + name=course_name, + title=course_title, + enrollment_start=enrollment_start, + enrollment_end=enrollment_end, + valid_until=valid_until, + is_restricted=False, + ) + course_objects.append(course) + + # sections for this course + for _ in range(random.randint(min_sections, max_sections)): + mentor_user = UserFactory.build() + user_objects.append(mentor_user) + mentor = MentorFactory.build(course=course, user=mentor_user) + mentor_objects.append(mentor) + section = SectionFactory.build( + mentor=mentor, capacity=random.randint(min_students, max_students) + ) + section_objects.append(section) + + # create spacetimes and section occurrences; use realistic spacetimes as well + cur_spacetimes, cur_section_occurrences = build_spacetimes_occurrences_for( + section, true_random_spacetimes=False + ) + spacetime_objects.extend(cur_spacetimes) + section_occurrence_objects.extend(cur_section_occurrences) + + # create students for the section + student_users = UserFactory.build_batch( + random.randint(min_students, section.capacity) + ) user_objects.extend(student_users) students = [] for student_user in student_users: @@ -495,7 +624,20 @@ def generate_test_data(preconfirm=False): ) student_objects.extend(students) - print("Saving models to database...") + print(f"({(time.perf_counter_ns() - _create_models_start)/1e6:.6f} ms)") + + # randomize the order of object creation + random.shuffle(user_objects) + random.shuffle(course_objects) + random.shuffle(mentor_objects) + random.shuffle(section_objects) + random.shuffle(spacetime_objects) + random.shuffle(section_occurrence_objects) + random.shuffle(student_objects) + + print("Saving models to database... ", end="") + _save_models_start = time.perf_counter_ns() + User.objects.bulk_create(user_objects) Course.objects.bulk_create(course_objects) Mentor.objects.bulk_create(mentor_objects) @@ -504,6 +646,8 @@ def generate_test_data(preconfirm=False): SectionOccurrence.objects.bulk_create(section_occurrence_objects) Student.objects.bulk_create(student_objects) + print(f"({(time.perf_counter_ns() - _save_models_start)/1e6:.6f} ms)") + # reload students, fetching related fields to optimize attendance creation prefetched_student_objects = Student.objects.select_related( "section", "course" @@ -512,13 +656,24 @@ def generate_test_data(preconfirm=False): "attendance_set__sectionoccurrence", ) - print("Generating attendances...") + print("Generating attendances... ", end="") + _generate_attendances_start = time.perf_counter_ns() + attendance_objects = [] for student in prefetched_student_objects: attendance_objects.extend(build_attendances_for(student)) + + # shuffle attendance objects + random.shuffle(attendance_objects) + + # create attendance objects Attendance.objects.bulk_create(attendance_objects) - print("Generating resources...") + print(f"({(time.perf_counter_ns() - _generate_attendances_start)/1e6:.6f} ms)") + + print("Generating resources... ", end="") + _generate_resources_start = time.perf_counter_ns() + resource_objects = [] for course in course_objects: # don't create resources for restricted courses @@ -532,8 +687,15 @@ def generate_test_data(preconfirm=False): course=course, week_num=i + 1, date=date, topics=f"Topic {val}" ) ) + + # randomize resource objects + random.shuffle(resource_objects) + + # create resource objects Resource.objects.bulk_create(resource_objects) - print("Done.\n") + print(f"({(time.perf_counter_ns() - _generate_resources_start)/1e6:.6f} ms)") + + print(f"Done. ({(time.perf_counter_ns() - _start_time)/1e6:.6f} ms)\n") create_demo_account()