From 778d27c574723cc49a9644602353dd24f384a0e1 Mon Sep 17 00:00:00 2001 From: Ayanami Date: Sun, 11 Aug 2024 17:39:20 +0800 Subject: [PATCH] fix: fix issues mentioned by code review --- cmd/load/extend_teacher_profile.go | 4 +- cmd/load/extend_training_plan.go | 5 +- handler/teacher_test.go | 2 +- model/converter/course.go | 2 +- model/converter/teacher.go | 2 +- model/converter/trainingplan.go | 4 +- model/domain/course.go | 20 ++---- model/dto/trainingplan.go | 6 +- model/po/course.go | 8 +-- model/po/teacher.go | 20 +++--- repository/trainingplan.go | 10 +-- service/trainingplan.go | 2 +- util/selenium-get/common.py | 53 +++++++++++++++ util/selenium-get/load2db.go | 61 ++++++++---------- util/selenium-get/requirements.txt | 3 + util/selenium-get/teacher-profile.py | 57 +++++++---------- util/selenium-get/training-plan.py | 96 ++++++++++++---------------- 17 files changed, 177 insertions(+), 178 deletions(-) create mode 100644 util/selenium-get/common.py create mode 100644 util/selenium-get/requirements.txt diff --git a/cmd/load/extend_teacher_profile.go b/cmd/load/extend_teacher_profile.go index 759e8ef..a8514d3 100644 --- a/cmd/load/extend_teacher_profile.go +++ b/cmd/load/extend_teacher_profile.go @@ -31,7 +31,7 @@ func main() { if len(teachers) == 1 { teachers[0].Email = t.Mail teachers[0].ProfileURL = t.ProfileUrl - teachers[0].ProfileDesc = t.ProfileDescription + teachers[0].Biography = t.Biography teachers[0].Picture = t.HeadImage db.Save(&teachers[0]) continue @@ -43,7 +43,7 @@ func main() { if tt.Department == t.Department { tt.Email = t.Mail tt.ProfileURL = t.ProfileUrl - tt.ProfileDesc = t.ProfileDescription + tt.Biography = t.Biography tt.Picture = t.HeadImage db.Save(&tt) confirm = true diff --git a/cmd/load/extend_training_plan.go b/cmd/load/extend_training_plan.go index 105770d..cd44ded 100644 --- a/cmd/load/extend_training_plan.go +++ b/cmd/load/extend_training_plan.go @@ -41,7 +41,7 @@ func main() { Department: tp.Department, EntryYear: strconv.Itoa(tp.EntryYear), TotalYear: tp.TotalYear, - MinPoints: tp.MinPoints, + MinCredits: tp.MinCredits, MajorCode: tp.Code, MajorClass: tp.MajorClass, } @@ -63,8 +63,7 @@ func main() { tpc_po := po.TrainingPlanCoursePO{ TrainingPlanID: int64(tp_po.ID), CourseID: int64(course.ID), - SuggestYear: int64(c.SuggestYear), - SuggestSemester: int64(c.SuggestSemester), + SuggestSemester: c.SuggestSemester, Department: c.Department, } cresult = db.Model(po.TrainingPlanCoursePO{}).Create(&tpc_po) diff --git a/handler/teacher_test.go b/handler/teacher_test.go index 105ad8d..3620061 100644 --- a/handler/teacher_test.go +++ b/handler/teacher_test.go @@ -34,7 +34,7 @@ func TestSearchTeacherListHandler(t *testing.T) { }{ "valid-multi-req": {"name=古金宇&page=1&page_size=3", http.StatusOK}, "valid-single-req": {"name=古金宇&department=电子信息与电气工程学院&page=1&page_size=3", http.StatusOK}, - "invalid-value-req": {"name=你谁", http.StatusBadRequest}, + "invalid-value-req": {"name=你谁", http.StatusOK}, "invalid-page-req": {"page=-1&page_size=10086", http.StatusOK}, "overflow-page-req": {"page=10086&page_size=1", http.StatusOK}, "invalid-key-req": {"hello=1", http.StatusBadRequest}, diff --git a/model/converter/course.go b/model/converter/course.go index 263edab..db12f14 100644 --- a/model/converter/course.go +++ b/model/converter/course.go @@ -183,7 +183,7 @@ func ConvertTrainingPlanPOToDomain(trainingPlan po.TrainingPlanPO) domain.Traini EntryYear: trainingPlan.EntryYear, MajorCode: trainingPlan.MajorCode, MajorClass: trainingPlan.MajorClass, - MinPoints: trainingPlan.MinPoints, + MinCredits: trainingPlan.MinCredits, TotalYear: int(trainingPlan.TotalYear), } } diff --git a/model/converter/teacher.go b/model/converter/teacher.go index 39850f7..90ed02e 100644 --- a/model/converter/teacher.go +++ b/model/converter/teacher.go @@ -20,7 +20,7 @@ func ConvertTeacherPOToDomain(teacher *po.TeacherPO) *domain.Teacher { Title: teacher.Title, Picture: teacher.Picture, ProfileURL: teacher.ProfileURL, - ProfileDesc: teacher.ProfileDesc, + ProfileDesc: teacher.Biography, } } diff --git a/model/converter/trainingplan.go b/model/converter/trainingplan.go index 1034e37..7e42aae 100644 --- a/model/converter/trainingplan.go +++ b/model/converter/trainingplan.go @@ -13,7 +13,6 @@ func ConvertTrainingPlanCourseDomainToDTO(courseDomain domain.TrainingPlanCourse Name: courseDomain.Name, Code: courseDomain.Code, Credit: courseDomain.Credit, - SuggestYear: courseDomain.SuggestYear, SuggestSemester: courseDomain.SuggestSemester, } } @@ -31,7 +30,7 @@ func ConvertTrainingPlanDomainToDTO(domain domain.TrainingPlanDetail) dto.Traini Department: domain.Department, EntryYear: int64(entryYear), MajorName: domain.Major, - MinPoints: domain.MinPoints, + MinCredits: domain.MinCredits, MajorClass: domain.MajorClass, TotalYear: int64(domain.TotalYear), Courses: courses, @@ -52,7 +51,6 @@ func ConvertTrainingPlanCoursePOToDomain(coursePO po.TrainingPlanCoursePO, baseC Name: baseCoursePO.Name, Credit: baseCoursePO.Credit, SuggestSemester: coursePO.SuggestSemester, - SuggestYear: coursePO.SuggestYear, Department: coursePO.Department, } } diff --git a/model/domain/course.go b/model/domain/course.go index d396c9e..d283e6d 100644 --- a/model/domain/course.go +++ b/model/domain/course.go @@ -41,11 +41,11 @@ type TrainingPlan struct { type TrainingPlanDetail struct { ID int64 Major string - Department string - EntryYear string MajorCode string MajorClass string - MinPoints float64 + Department string + EntryYear string + MinCredits float64 TotalYear int Courses []TrainingPlanCourse } @@ -54,21 +54,9 @@ type TrainingPlanCourse struct { Code string Name string Credit float64 - SuggestYear int64 - SuggestSemester int64 + SuggestSemester string Department string } -type TrainingPlanRateInfo struct { - Avg float64 - Count int64 - TrainingPlanID int64 - Rates []TrainingPlanRate -} -type TrainingPlanRate struct { - TrainingPlanID int64 - UserID int64 - Rate int64 -} type TrainingPlanFilter struct { Page int64 diff --git a/model/dto/trainingplan.go b/model/dto/trainingplan.go index 8e33fb2..6a493ec 100644 --- a/model/dto/trainingplan.go +++ b/model/dto/trainingplan.go @@ -5,8 +5,7 @@ type TrainingPlanCourseDTO struct { Code string `json:"code"` Name string `json:"name"` Credit float64 `json:"credit"` - SuggestYear int64 `json:"suggest_year"` - SuggestSemester int64 `json:"suggest_semester"` + SuggestSemester string `json:"suggest_semester"` Department string `json:"department"` } @@ -14,12 +13,11 @@ type TrainingPlanListItemDTO struct { ID int64 `json:"id"` Code string `json:"code"` MajorName string `json:"name"` - MinPoints float64 `json:"min_points"` + MinCredits float64 `json:"min_credits"` MajorClass string `json:"major_class"` EntryYear int64 `json:"entry_year"` Department string `json:"department"` TotalYear int64 `json:"total_year"` - Grade float32 `json:"grade"` Degree string `json:"degree"` Courses []TrainingPlanCourseDTO `json:"courses"` } diff --git a/model/po/course.go b/model/po/course.go index d7f8aa7..4d78281 100644 --- a/model/po/course.go +++ b/model/po/course.go @@ -72,7 +72,7 @@ type TrainingPlanPO struct { EntryYear string `gorm:"index;index:uniq_training_plan,unique"` //==Grade,年级 MajorCode string `gorm:"index;index:uniq_training_plan,unique"` TotalYear int `gorm:"index;index:uniq_training_plan,unique"` - MinPoints float64 `gorm:"index;index:uniq_training_plan,unique"` + MinCredits float64 `gorm:"index;index:uniq_training_plan,unique"` MajorClass string `gorm:"index;index:uniq_training_plan,unique"` // 专业类 } @@ -84,10 +84,8 @@ type TrainingPlanCoursePO struct { gorm.Model CourseID int64 `gorm:"index;index:uniq_training_plan_course,unique"` TrainingPlanID int64 `gorm:"index;index:uniq_training_plan_course,unique"` - //SuggestTime string `gorm:"index;index:uniq_training_plan_course,unique"` - // 推荐修读时间:学年+学期，如 2023-2024-2 - SuggestYear int64 `gorm:"index;index:uniq_training_plan_course,unique"` - SuggestSemester int64 `gorm:"index;index:uniq_training_plan_course,unique"` + // SuggestSemester:学年+学期，如 2023-2024-2 + SuggestSemester string `gorm:"index;index:uniq_training_plan_course,unique"` Department string `gorm:"index;"` } diff --git a/model/po/teacher.go b/model/po/teacher.go index 7fa2830..11a5a5d 100644 --- a/model/po/teacher.go +++ b/model/po/teacher.go @@ -4,16 +4,16 @@ import "gorm.io/gorm" type TeacherPO struct { gorm.Model - Name string `gorm:"index"` - Code string `gorm:"index:,unique"` - Email string `gorm:"index"` - Department string `gorm:"index"` - Title string - Pinyin string `gorm:"index"` - PinyinAbbr string `gorm:"index"` - Picture string // picture URL - ProfileURL string - ProfileDesc string + Name string `gorm:"index"` + Code string `gorm:"index:,unique"` + Email string `gorm:"index"` + Department string `gorm:"index"` + Title string + Pinyin string `gorm:"index"` + PinyinAbbr string `gorm:"index"` + Picture string // picture URL + ProfileURL string + Biography string // 个人简述 } func (po *TeacherPO) TableName() string { diff --git a/repository/trainingplan.go b/repository/trainingplan.go index 67fe092..d74481d 100644 --- a/repository/trainingplan.go +++ b/repository/trainingplan.go @@ -138,17 +138,11 @@ type ITrainingPlanCourseQuery interface { WithTrainingPlanID(trainingPlanID int64) DBOption WithCourseID(courseID int64) DBOption WithCourseIDs(courseIDs []int64) DBOption - WithSuggestSemester(semester int64) DBOption - WithSuggestYear(year int64) DBOption + WithSuggestSemester(semester string) DBOption WithDepartment(department string) DBOption } -func (t *TrainingPlanCourseQuery) WithSuggestYear(year int64) DBOption { - return func(db *gorm.DB) *gorm.DB { - return db.Where("suggest_year = ?", year) - } -} -func (t *TrainingPlanCourseQuery) WithSuggestSemester(semester int64) DBOption { +func (t *TrainingPlanCourseQuery) WithSuggestSemester(semester string) DBOption { return func(db *gorm.DB) *gorm.DB { return db.Where("suggest_semester = ?", semester) } diff --git a/service/trainingplan.go b/service/trainingplan.go index 6ed426f..7410a4d 100644 --- a/service/trainingplan.go +++ b/service/trainingplan.go @@ -96,7 +96,7 @@ func SearchTrainingPlanList(ctx context.Context, filter domain.TrainingPlanFilte func GetTrainingPlanListByIDs(ctx context.Context, trainingPlanIDs []int64) (map[int64]domain.TrainingPlanDetail, error) { - domainTrainingPlans := make(map[int64]domain.TrainingPlanDetail, 0) + domainTrainingPlans := make(map[int64]domain.TrainingPlanDetail) for _, id := range trainingPlanIDs { data, err := GetTrainingPlanDetail(ctx, id) if err != nil { diff --git a/util/selenium-get/common.py b/util/selenium-get/common.py new file mode 100644 index 0000000..fcdea0e --- /dev/null +++ b/util/selenium-get/common.py @@ -0,0 +1,53 @@ +import requests +from typing import Callable +from selenium.webdriver.common.by import By +from argparse import ArgumentParser +from selenium.webdriver.support import expected_conditions as EC +from selenium.webdriver.support.ui import WebDriverWait +from selenium import webdriver +from selenium.webdriver.chrome.service import Service +class Automator: + timeout = 60 + def __init__(self, description:str="") -> None: + # automate downloading chrome driver if + # 1.you haven't installed chrome + # 2.imcompatible with your browser + service = Service() + options = webdriver.ChromeOptions() + self.driver = webdriver.Chrome(service=service, options=options) + self.parser = ArgumentParser(description=description) + self.parser.add_argument("-n", "--name", type=str, action="store", help="jacount for login") + self.parser.add_argument("-p", "--password", type=str, action="store", help="password for login") + self.parser.add_argument("-d","--debug", action="store_true", help="In debug mode, the program will not save the result.") + self.parser.add_argument("-v", "--verbose", action="store_true", help="Verbose mode") + self.has_login = False + def login(self, name:str, password:str): + login_url = "https://i.sjtu.edu.cn/jaccountlogin" + self.driver.implicitly_wait(time_to_wait=self.timeout) + self.driver.get(url=login_url) + username_input = self.driver.find_element(By.ID, "input-login-user") + password_input = self.driver.find_element(By.ID, "input-login-pass") + username_input.send_keys("{}".format(name)) + password_input.send_keys("{}".format(password)) + # driver -> bool + cond:Callable[[object], bool] = EC.url_contains("https://i.sjtu.edu.cn/xtgl") + WebDriverWait(self.driver, self.timeout).until(cond) + session = requests.Session() + # 获取登录后的 Cookie + cookies = self.driver.get_cookies() + + # 将 Cookie 转换为 requests 库可以使用的格式 + for cookie in cookies: + session.cookies.set(cookie['name'], cookie['value']) + self.session = session + self.has_login = True + def gen_driver(self): + return self.driver + + def end(self): + self.driver.close() + + def get_session(self): + if not self.login: + raise Exception("Please login first") + return self.session diff --git a/util/selenium-get/load2db.go b/util/selenium-get/load2db.go index 9aff5ac..5b317c3 100644 --- a/util/selenium-get/load2db.go +++ b/util/selenium-get/load2db.go @@ -4,7 +4,9 @@ import ( "bufio" "encoding/json" "errors" + "fmt" "jcourse_go/model/po" + "log" "os" "strconv" "strings" @@ -17,14 +19,13 @@ type LoadedCourse struct { Name string Credit float32 Department string - SuggestYear int - SuggestSemester int + SuggestSemester string } type LoadedTrainingPlan struct { Code string Name string TotalYear int - MinPoints float64 + MinCredits float64 MajorClass string Department string EntryYear int @@ -38,8 +39,8 @@ func LoadedCourse2PO(course LoadedCourse) (po.BaseCoursePO, po.TrainingPlanCours Name: course.Name, Credit: float64(course.Credit), }, po.TrainingPlanCoursePO{ - SuggestYear: int64(course.SuggestYear), - SuggestSemester: int64(course.SuggestSemester), + SuggestSemester: course.SuggestSemester, + Department: course.Department, } } func Line2Course(line string) LoadedCourse { @@ -52,20 +53,12 @@ func Line2Course(line string) LoadedCourse { if err != nil { credit = 0.0 } - suggest_year, err := strconv.ParseInt(meta[3][0:4], 10, 32) - if err != nil { - suggest_year = 0 - } - suggest_semester, err := strconv.Atoi(meta[4]) - if err != nil { - suggest_semester = 0 - } + suggest_semester := fmt.Sprintf("%s-%s", meta[3], meta[4]) // 2018-2019-1 return LoadedCourse{ Code: meta[0], Name: meta[1], Credit: float32(credit), - SuggestYear: int(suggest_year), - SuggestSemester: int(suggest_semester), + SuggestSemester: suggest_semester, Department: meta[5], } } @@ -81,7 +74,7 @@ func Lines2TrainingPlan(lines []string) LoadedTrainingPlan { plan.Name = metaInfo[2] plan.Code = metaInfo[0] plan.TotalYear, _ = strconv.Atoi(metaInfo[4]) - plan.MinPoints, _ = strconv.ParseFloat(metaInfo[5], 64) + plan.MinCredits, _ = strconv.ParseFloat(metaInfo[5], 64) plan.MajorClass = metaInfo[6] plan.Department = metaInfo[7] plan.Degree = metaInfo[8] @@ -98,11 +91,10 @@ func TrainingPlan2PO(plan LoadedTrainingPlan) po.TrainingPlanPO { Department: plan.Department, EntryYear: strconv.Itoa(plan.EntryYear), TotalYear: plan.TotalYear, - MinPoints: plan.MinPoints, + MinCredits: plan.MinCredits, MajorCode: plan.Code, MajorClass: plan.MajorClass, } - // TODO: change po } func SaveTrainingPlanCourses(plan LoadedTrainingPlan, db *gorm.DB, tid int64) { if db == nil { @@ -156,7 +148,6 @@ func LoadTrainingPLans(from string) []LoadedTrainingPlan { var allTrainingPlans []LoadedTrainingPlan for scanner.Scan() { text := scanner.Text() - // println(text) if text == "" { if len(lines) > 0 { allTrainingPlans = append(allTrainingPlans, Lines2TrainingPlan(lines)) @@ -188,16 +179,16 @@ type LoadedTeacher struct { "mail": "wuziyun@vip.qq.com", "profile_description": "PI\u7b80\u4ecb\uff1a\u5434\u7d2b\u4e91\u535a\u58eb\uff0c\u535a\u5bfc\uff0c\u6e56\u5357\u5e38\u5fb7\u4eba\uff0c2014\u5e74\u6bd5\u4e1a\u4e8e\u65b0\u52a0\u5761..." },*/ - Name string `json:"name"` - Code int64 `json:"code"` - Department string `json:"department"` - Title string `json:"title"` - Pinyin string `json:"pinyin"` - PinyinAbbr string `json:"pinyin_abbr"` - ProfileUrl string `json:"profile_url"` - HeadImage string `json:"head_image"` - Mail string `json:"mail"` - ProfileDescription string `json:"profile_description"` + Name string `json:"name"` + Code int64 `json:"code"` + Department string `json:"department"` + Title string `json:"title"` + Pinyin string `json:"pinyin"` + PinyinAbbr string `json:"pinyin_abbr"` + ProfileUrl string `json:"profile_url"` + HeadImage string `json:"head_image"` + Mail string `json:"mail"` + Biography string `json:"biography"` } func Teacher2PO(teacher LoadedTeacher) po.TeacherPO { @@ -209,10 +200,9 @@ func Teacher2PO(teacher LoadedTeacher) po.TeacherPO { Title: teacher.Title, Pinyin: teacher.Pinyin, PinyinAbbr: teacher.PinyinAbbr, - // TODO: add more - ProfileDesc: teacher.ProfileDescription, - ProfileURL: teacher.ProfileUrl, - Picture: teacher.HeadImage, + Biography: teacher.Biography, + ProfileURL: teacher.ProfileUrl, + Picture: teacher.HeadImage, } } func SaveTeacher(teachers []LoadedTeacher, db *gorm.DB) { @@ -244,6 +234,9 @@ func LoadTeacherProfiles(from string) []LoadedTeacher { defer file.Close() data, _ := os.ReadFile(from) var teachers []LoadedTeacher - json.Unmarshal(data, &teachers) + err = json.Unmarshal(data, &teachers) + if err != nil { + log.Fatalf("In read teacher profile from file %#v:%#v", from, err) + } return teachers } diff --git a/util/selenium-get/requirements.txt b/util/selenium-get/requirements.txt new file mode 100644 index 0000000..fe2064e --- /dev/null +++ b/util/selenium-get/requirements.txt @@ -0,0 +1,3 @@ +pypinyin==0.50.0 +requests==2.31.0 +selenium==4.23.1 diff --git a/util/selenium-get/teacher-profile.py b/util/selenium-get/teacher-profile.py index 6fe71f9..fcee35c 100644 --- a/util/selenium-get/teacher-profile.py +++ b/util/selenium-get/teacher-profile.py @@ -2,17 +2,9 @@ import requests import json import pprint -import time from pypinyin import lazy_pinyin from typing import List -from selenium import webdriver -from selenium.webdriver.common.by import By -from argparse import ArgumentParser -driver = webdriver.Chrome() -session = requests.Session() -parser = ArgumentParser(description="A simple scrapy to download the teacher profile and save as json.") -base_url = "https://faculty.sjtu.edu.cn/" json_indent = 4 """ type TeacherPO struct { @@ -80,19 +72,7 @@ def resp_to_teacher(resp: dict)->Teacher: return teacher -def login(name:str, password:str): - driver.get(url="https://i.sjtu.edu.cn/jaccountlogin") - username_input = driver.find_element(By.ID, "input-login-user") - password_input = driver.find_element(By.ID, "input-login-pass") - username_input.send_keys("{}".format(name)) - password_input.send_keys("{}".format(password)) - time.sleep(10) -def set_cookie(): - cookies = driver.get_cookies() - # 转化为 requests 库可以使用的格式 - for cookie in cookies: - session.cookies.set(cookie['name'], cookie['value']) -def get_teacher_list(_print:bool=False, limit:int=10000)->List[Teacher]: +def get_teacher_list(session:requests.Session, _print:bool=False, limit:int=10000)->List[Teacher]: url = f"{base_url}/system/resource/tsites/advancesearch.jsp" query_param = { "collegeid": 0, @@ -142,35 +122,43 @@ def append_json(data: List[Teacher], to="./data/teachers.json"): teachers.extend([t.to_dict() for t in data]) with open(to, "w") as f: json.dump(teachers, f, indent=json_indent) -def automation(): - parser.add_argument("-n", "--name", type=str, action="store", help="jacount for login") - parser.add_argument("-p", "--password", type=str, action="store", help="password for login") - parser.add_argument("-d","--debug", action="store_true", help="In debug mode, the program will not save the result to the file.") + +if __name__=="__main__": + description = "A simple scrapy to download the teacher profile and save as json." + if __package__ is None: + import sys + from os import path + sys.path.append(path.dirname(path.dirname( path.abspath(__file__)))) + from common import Automator + else: + from .common import Automator + automator = Automator( + description=description + ) + base_url = "https://faculty.sjtu.edu.cn/" + parser = automator.parser parser.add_argument("-l", "--load_from", type=str, action="store", help="Load the result from the json file.") parser.add_argument("-a", "--append_from", type=str, action="store", help="Append the result from the json file.") parser.add_argument("-s", "--save_to", type=str, action="store", help="Save the result to the json file.") parser.add_argument("--limit", type=int, help="The maximum number of request teachers") - parser.add_argument("-v", "--verbose", action="store_true", help="Verbose mode") -def main(): - automation() args = parser.parse_args() init_teachers = [] if not args.name or not args.password: print("jacount and password are required") exit(0) + automator.login(args.name, args.password) + session = automator.get_session() if args.load_from and os.path.exists(args.load_from): with open(args.load_from, "r") as f: init_teachers = json.load(f) if args.append_from and os.path.exists(args.append_from): with open(args.append_from, "r") as f: init_teachers.extend(json.load(f)) - - login(name=args.name, password=args.password) - set_cookie() + print("开始查询，请稍等.......") if args.limit: - new_teachers = get_teacher_list(args.verbose, args.limit) + new_teachers = get_teacher_list(session=session,_print=args.verbose, limit=args.limit) else: - new_teachers = get_teacher_list(args.verbose) + new_teachers = get_teacher_list(session=session,_print=args.verbose) all_teachers = init_teachers + new_teachers print("当前教师数量：{}".format(len(all_teachers))) if not args.debug: @@ -178,6 +166,5 @@ def main(): save_json(all_teachers, to=args.save_to) else: save_json(all_teachers) + automator.end() -main() -driver.close() \ No newline at end of file diff --git a/util/selenium-get/training-plan.py b/util/selenium-get/training-plan.py index f60e404..73bab99 100644 --- a/util/selenium-get/training-plan.py +++ b/util/selenium-get/training-plan.py @@ -1,11 +1,8 @@ from copy import deepcopy import os -from typing import List, Dict -from selenium import webdriver -from selenium.webdriver.common.by import By +from typing import List, Dict, Callable + import requests -import time -import argparse class Major: @@ -167,54 +164,36 @@ class TrainingPlan: def __init__(self, major: Major, year): self.major = major self.year = year - self.courses = [] + self.courses:List[Course] = [] def __csv__(self): courses_csv = "\n".join([str(c) for c in self.courses]) return f"{self.major.__csv__()},{self.year}\n{courses_csv}\n" - def add_course(self, course): + def add_course(self, course: Course): self.courses.append(deepcopy(course)) def __str__(self) -> str: return self.__csv__() - -# 初始化 Selenium WebDriver -driver = webdriver.Chrome() -session = requests.Session() - - -def login(jaccount_name: str = "", password: str = ""): - driver.get(url="https://i.sjtu.edu.cn/jaccountlogin") - username_input = driver.find_element(By.ID, "input-login-user") - password_input = driver.find_element(By.ID, "input-login-pass") - username_input.send_keys("{}".format(jaccount_name)) - password_input.send_keys("{}".format(password)) - time.sleep(10) - # 手输一下验证码 - - # 获取登录后的 Cookie - cookies = driver.get_cookies() - - # 将 Cookie 转换为 requests 库可以使用的格式 - for cookie in cookies: - session.cookies.set(cookie['name'], cookie['value']) - - -def abstract_postlist(url: str, post_data: dict, callback): +def abstract_postlist(session:requests.Session, url: str, post_data: Dict, callback: Callable[[Dict], None]): response = session.post(url, data=post_data) if response.status_code == 200: print("请求成功") - data = response.json()['items'] - for d in data: - callback(deepcopy(d)) + try: + data = response.json()['items'] + for d in data: + callback(deepcopy(d)) + except Exception as e: + print(response.text) + print(e) + else: print(f"请求失败，状态码: {response.status_code}") print(response.text) -def get_training_plans(tps: List[TrainingPlan]): +def get_training_plans(session: requests.Session, tps: List[TrainingPlan]): url = "https://i.sjtu.edu.cn/jxzxjhgl/jxzxjhck_cxJxzxjhckIndex.html?doType=query&gnmkdm=N153540" data = { "jg_id": "", @@ -231,7 +210,7 @@ def get_training_plans(tps: List[TrainingPlan]): "time": "1" } - def generate_training_plan(tp: dict): + def generate_training_plan(tp: Dict)->None: required_keys = ['zyh', 'zymc', 'zyh_id', 'jxzxjhxx_id', 'jg_id'] min_points = "0" if 'zdxf' in tp: @@ -249,7 +228,7 @@ def generate_training_plan(tp: dict): tps.append(TrainingPlan(deepcopy(_major), tp['njmc'])) callback = generate_training_plan - abstract_postlist(url, data, callback) + abstract_postlist(session=session, url=url, post_data=data, callback=callback) """ @@ -268,13 +247,13 @@ def generate_training_plan(tp: dict): "kcmc" //课程名称 "xf" //学分(不一定有) "kkbmmc" //开课学院名称(不一定有) -"jyxdxnm" //建议修读学年 -"jyxdxqm" //建议修读学期 +"jyxdxnm" //建议修读学年 (2018-2019格式) +"jyxdxqm" //建议修读学期 (1,2,3) "" """ -def get_training_plan_courses(tps: List[TrainingPlan]): +def get_training_plan_courses(session: requests.Session, tps: List[TrainingPlan]): # 构建请求的 URL 和数据 url = "https://i.sjtu.edu.cn/jxzxjhgl/jxzxjhkcxx_cxJxzxjhkcxxIndex.html?doType=query&gnmkdm=N153540" for tp in tps: @@ -311,10 +290,10 @@ def callback(d: Dict): suggest_semester=d['jyxdxqm'], department=d['kkbmmc'])) - abstract_postlist(url, data, callback) + abstract_postlist(session=session, url=url, post_data=data, callback=callback) -def save_csv(tps: List[TrainingPlan], to="./data/trainingPlan.txt", +def save_csv(tps: List[TrainingPlan], to:str="./data/trainingPlan.txt", append=False): dir = os.path.dirname(to) if not os.path.exists(dir): @@ -326,15 +305,24 @@ def save_csv(tps: List[TrainingPlan], to="./data/trainingPlan.txt", for tp in tps: f.write(tp.__csv__() + "\n") - -parser = argparse.ArgumentParser() -parser.add_argument('-n', "--name", help="jaccount name", default="") -parser.add_argument('-p', "--password", help="password", default="") -args = parser.parse_args() -TrainingPlans: List[TrainingPlan] = [] -login(args.name, args.password) -get_training_plans(TrainingPlans) -get_training_plan_courses(TrainingPlans) -save_csv(TrainingPlans) -# 关闭 Selenium WebDriver -driver.quit() +if __name__ == "__main__": + if __package__ is None: + import sys + from os import path + sys.path.append(path.dirname( path.dirname(path.abspath(__file__)))) + from common import Automator + else: + from .common import Automator + automator = Automator(description="Get training-plans") + parser = automator.parser + args = parser.parse_args() + trainingPlans: List[TrainingPlan] = [] + automator.login(args.name, args.password) + session = automator.get_session() + get_training_plans(session=session, tps=trainingPlans) + get_training_plan_courses(session=session,tps=trainingPlans) + if not args.debug: + save_csv(trainingPlans) + else: + print(trainingPlans[:10]) + automator.end()