diff --git a/.gitignore b/.gitignore index 6669d92..a37fea3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ /__pycache__ /.idea +geckodriver.log +/my-run.sh diff --git a/code/.gitignore b/code/.gitignore index 749e461..c015bd0 100644 --- a/code/.gitignore +++ b/code/.gitignore @@ -1,2 +1,3 @@ -/*.log -/env \ No newline at end of file +/env +/my-run.sh +/__pycache__ diff --git a/code/insert_data.py b/code/insert_data.py index 2db350f..72527e2 100644 --- a/code/insert_data.py +++ b/code/insert_data.py @@ -2,6 +2,7 @@ import datetime import re import argparse +from mylogging import logger from selenium import webdriver import pandas as pd @@ -14,6 +15,7 @@ date_regex = re.compile("^[0-9]+\.[0-9]+\.[0-9]+$") + # PARSE DATA def parse_data(file): @@ -26,34 +28,41 @@ def parse_data(file): class DataInserter: - def __init__(self, data): + def __init__(self, data, disable_all=False, max_tries=10): self.data = data self.driver = None - self.max_tries = 10 + self.max_tries = max_tries + self.disable_all = disable_all + self.logged_in = False def navigate_to_page(self): sport_db_url = 'https://www.sportdb.ch' + + logger.debug('Navigating to %s', sport_db_url) + try: self.driver = webdriver.Firefox() except WebDriverException: - print('Could not run firefox locally. Switching to remote option.') + logger.info('Could not run firefox locally. Switching to remote option.') n_tries = 0 while self.driver is None: try: self.driver = webdriver.Remote("http://localhost:4444/wd/hub", DesiredCapabilities.FIREFOX) - print('Successfully opened sportdb') + logger.info('Successfully opened sportdb') except MaxRetryError: n_tries += 1 - print('Remote webdriver is not running yet ({}/{})...'.format(n_tries, self.max_tries)) + logger.warning('Remote webdriver is not running yet ({}/{})...'.format(n_tries, self.max_tries)) if n_tries >= self.max_tries: raise Exception('Remote webdriver does not seem to be running...') else: time.sleep(2) self.driver.get(sport_db_url) + logger.debug('Navigated to %s', sport_db_url) def login(self, username, password): + logger.debug('Filling out login form...') username_field = self.driver.find_element_by_id('j_username') username_field.clear() username_field.send_keys(username) @@ -62,101 +71,145 @@ def login(self, username, password): username_field.clear() username_field.send_keys(password) + logger.debug('Clicking login button') login = self.driver.find_element_by_id('ButtonLogin') login.click() + self.logged_in = True + logger.debug('Clicked login button') + src = self.driver.page_source if 'Bitte überprüfen Sie Benutzername und Passwort' in src or 'Bitte Benutzername und Passwort angeben' in src: raise Exception('Something went wrong. Most likely, you provided the wrong username or password') def to_awk(self, course_id): + logger.debug('Browsing to AWK with course id %s...', course_id) if course_id is None: + logger.debug('Mode: manual') input('No course_id provided. Manually navigate to "Anwesenheitskontrolle" for your course.') else: + logger.debug('Mode: automatic') self.driver.get('https://www.sportdb.ch/extranet/kurs/kursEditAwk.do?kursId={}'.format(course_id)) + + logger.debug('Waiting a bit for page to fully load') + time.sleep(1) + if 'Error' in self.driver.title: raise Exception('Something went wrong. Most likely, you entered the wrong course id.') + logger.debug('Browsed to AWK...') - @staticmethod - def set_attendance(attended, box, name, date): - + def set_attendance(self, attended, box, name, date): + if attended: + logger.debug("Attended") + attended = attended and not self.disable_all + logger.debug('Setting attendance for %s on %s', name, date) if attended and not box.is_selected(): - print('{} attended on {}'.format(name, date)) + logger.debug('{} attended on {}'.format(name, date)) box.click() return True - if not attended and box.is_selected(): - print('{} did not attend on {}'.format(name, date)) + elif not attended and box.is_selected(): + logger.debug('{} did not attend on {}'.format(name, date)) box.click() return True + elif attended and box.is_selected(): + logger.debug('{} attended on {} (already entered)'.format(name, date)) + elif not attended and not box.is_selected(): + logger.debug('{} did not attend on {} (already entered)'.format(name, date)) + else: + logger.error('Program error') + assert False return False def enter_data(self): - changed = False + any_changed = False # match ids and days + logger.debug('Determining days on the current page') days = self.driver.find_elements_by_xpath(".//*[contains(@class, 'awkDay')]//span") days = [d.text for d in days if date_regex.match(d.text)] + logger.debug('Determining day ids on the current page') day_ids = self.driver.find_elements_by_xpath(".//*[contains(@class, 'select-all leiter')]") day_ids = [d.get_attribute('name') for d in day_ids] + logger.debug("Asserting length of results matches: \t\n%s, \t\n%s", days, day_ids) assert(len(days) == len(day_ids)) day_to_id = {day: day_id for day, day_id in zip(days, day_ids)} - print('Found days:', day_to_id) + logger.debug('Found days: %s', day_to_id) # enter data + logger.debug('Entering data...') for column in self.data: date = column.to_pydatetime().strftime('%d.%m.%Y') for key, val in self.data[column].iteritems(): js_id = key[0] - name = key[2] + last_name = key[1] + first_name = key[2] + name = first_name + ' ' + last_name + attended = val == 'x' if date in day_to_id: day_id = day_to_id[date] - box = self.driver.find_element_by_xpath( - ".//input[contains(@name, 'kursAktivitaetTeilnehmerMap({})')][contains(@name, 'I-{}')]" + path = ".//input[contains(@name, 'kursAktivitaetTeilnehmerMap({})')][contains(@value, 'I-{}')]"\ .format(day_id, js_id) - ) - changed = changed or self.set_attendance(attended, box, name, date) + logger.debug('Locating checkbox for %s (%s) on %s by path %s', name, js_id, date, path) + box = self.driver.find_element_by_xpath(path) + logger.debug('Filling out checkbox') + changed = self.set_attendance(attended, box, name, date) + any_changed = any_changed or changed + logger.debug('Filled out checkbox') + + logger.debug('Wait a bit for page to process changes') + time.sleep(1) # save - if changed: + if any_changed: + logger.debug('Saving data...') save = self.driver.find_element_by_id('formSave') save.click() + logger.debug('Saved data') + else: + logger.debug('Not saving, since there were no changes.') def to_previous(self): previous = self.driver.find_element_by_id('previousLink') c = previous.get_attribute("class") if 'disabled' not in c: - # reload to prevent stale elements (a bit of a hack) - previous = self.driver.find_element_by_id('previousLink') previous.click() + + logger.debug('Waiting a bit for page to fully load') + time.sleep(1) + return True else: return False def __del__(self): - if self.driver is not None: + if self.logged_in: + logger.debug('Logging out...') logout = self.driver.find_element_by_id('logout') logout.click() + logger.debug('Closing driver...') self.driver.close() -def run(data_file, username, password, course_id): +def run(data_file, username, password, course_id, disable_all): + logger.debug("Running...") + # parse data data = parse_data(data_file) # navigate - ins = DataInserter(data) + ins = DataInserter(data, disable_all) ins.navigate_to_page() ins.login(username, password) ins.to_awk(course_id) # enter data while True: - print('Entering data...') + logger.info('Entering data...') ins.enter_data() - print('Entered data. Going to previous page...') + logger.info('Entered data. Going to previous page...') more = ins.to_previous() if not more: break @@ -164,6 +217,8 @@ def run(data_file, username, password, course_id): if __name__ == "__main__": parser = argparse.ArgumentParser(description='Eintragehilfe für Anwesenheitskontrolle bei sportdb') + parser.add_argument('data_file', action='store', type=str, + help='File mit Daten. Siehe data/reference.xls für ein Referenzfile (letztes Argument)') parser.add_argument('--username', dest='username', action='store', type=str, help='Username für sportdb (z.B. js-123456)', required=True) parser.add_argument('--password', dest='password', action='store', @@ -171,9 +226,8 @@ def run(data_file, username, password, course_id): help='Passwort für sportdb (default: interaktive Eingabe)') parser.add_argument('--course-id', dest='course_id', action='store', default=None, type=str, help='Kurs ID (z.B. 1234567). Kann aus der URL der Anwesenheitskontrolle abgelesen werden. Wenn nicht angegeben, wirst du interaktiv angefragt, zur korrekten Anwesenheitskontrolle zu navigieren.') - parser.add_argument('--data-file', dest='data_file', action='store', - type=str, default='data/attendance.xls', - help='File mit Daten. Siehe data/reference.xls für ein Referenzfile') + parser.add_argument('--disable-all', dest='disable_all', action='store_true', default=False, + help='Deaktiviere die Anwesenheit für alle Personen und Daten im File.') args = parser.parse_args() if args.password is None: @@ -181,6 +235,6 @@ def run(data_file, username, password, course_id): else: password = args.password - run(args.data_file, args.username, password, args.course_id) + run(args.data_file, args.username, password, args.course_id, args.disable_all) diff --git a/code/log.ini b/code/log.ini deleted file mode 100644 index 6795dee..0000000 --- a/code/log.ini +++ /dev/null @@ -1,43 +0,0 @@ -[loggers] -keys=root,sLogger - -[handlers] -keys=fileHandler,consoleHandler,slackHandler - -[formatters] -keys=consoleFormatter,messageFormatter - -[logger_root] -level=DEBUG -handlers=consoleHandler - -[logger_sLogger] -level=DEBUG -handlers=fileHandler,consoleHandler,slackHandler -qualname=sLogger -propagate=0 - -[handler_consoleHandler] -class=StreamHandler -level=INFO -formatter=messageFormatter -args=(sys.stdout,) - -[handler_fileHandler] -class=FileHandler -level=DEBUG -formatter=consoleFormatter -args=('%(logfile)s',) - -[handler_slackHandler] -class=mylogging.SlackHandler -level=ERROR -formatter=messageFormatter -args=() - -[formatter_messageFormatter] -format=%(message)s - -[formatter_consoleFormatter] -format=[%(asctime)s] %(levelname)s [%(name)s.%(funcName)s:%(lineno)d] %(message)s -datefmt='%Y-%m-%d %H:%M:%S \ No newline at end of file diff --git a/code/logs/.gitignore b/code/logs/.gitignore new file mode 100644 index 0000000..890cdd8 --- /dev/null +++ b/code/logs/.gitignore @@ -0,0 +1 @@ +/*.log* diff --git a/code/mylogging.py b/code/mylogging.py index dc4c446..aa42f83 100644 --- a/code/mylogging.py +++ b/code/mylogging.py @@ -1,114 +1,35 @@ -from logging import * -from logging.config import fileConfig, dictConfig +import logging +from logging.handlers import RotatingFileHandler import os -import json -from collections import OrderedDict dir_path = os.path.dirname(os.path.realpath(__file__)) -log_path = os.path.join(dir_path, '../ci-output/script-logs') +log_path = os.path.join(dir_path, 'logs') log_file = os.path.join(log_path, 'logs.log') -log_ini_file = os.path.join(dir_path, 'log.ini') -def get_list(filename): - with open(filename, 'r', encoding='iso-8859-1') as f: - lines = f.read().splitlines() - lines = [l for l in lines if not l.startswith('#')] - return lines -class SlackHandler(Handler): - def __init__(self, *args, **kwargs): - super(SlackHandler, self).__init__(*args, **kwargs) - self.buffer = [] - self.slack_url = self.get_environment_variable('SLACK_TOKEN') - self.details = { - 'commit' : self.get_environment_variable('CI_COMMIT_SHA'), - 'message': self.get_environment_variable('CI_COMMIT_MESSAGE'), - 'user' : self.get_environment_variable('GITLAB_USER_NAME'), - 'user-email' : self.get_environment_variable('GITLAB_USER_EMAIL'), - 'job-URL': self.get_environment_variable('CI_JOB_URL') - } - if self.details['message'] is not None: - self.details['message'] = self.details['message'].rstrip() - for k in list(self.details.keys()): - if self.details[k] is None: - del self.details[k] - - # parse slack user IDs - self.slack_id_dict = {} - slack_ids_file = os.path.join(dir_path, 'slack-user-ids.txt') - ids_list = get_list(slack_ids_file) - for d in ids_list: - splits = d.split(",") - email = splits[0] - slack_id = splits[1] - self.slack_id_dict[email] = slack_id - - def get_environment_variable(self, name): - return os.environ.get(name, None) +def create_logger(): + logger = logging.getLogger("logger") + logger.setLevel(logging.DEBUG) - def emit(self, record): - msg = self.format(record) - logger.debug('Slack message: %s', msg) - self.buffer.append(msg) + formatter = logging.Formatter('[%(asctime)s] %(levelname)s [%(name)s.%(funcName)s:%(lineno)d] %(message)s') - def post_to_slack(self, msg): - if self.slack_url is None: - logger.info('Not sending message to slack because slack url is not set...') + # add a rotating handler + handler = RotatingFileHandler(log_file, backupCount=50) + handler.setLevel(logging.DEBUG) + handler.setFormatter(formatter) + logger.addHandler(handler) - else: - logger.info('Sending message to slack...') + # force rollover if files exists + if os.path.isfile(log_file): + handler.doRollover() - payload = json.dumps({'text': msg}) - logger.debug('Sending payload to slack %s: %s', self.slack_url, payload) - r = requests.post(self.slack_url, data=payload, headers={'Content-Type': 'application/json'}) - logger.debug("Slack Response: %s %s", r.status_code, r.text.encode('utf-8')) - - def format_slack_mention(self, user_email): - if user_email in self.slack_id_dict: - return "<@%s>" % self.slack_id_dict[user_email] - return "" + handler = logging.StreamHandler() + handler.setLevel(logging.INFO) + handler.setFormatter(formatter) + logger.addHandler(handler) - def flush(self): - if len(self.buffer) > 0: - labels = OrderedDict([ - ('commit', 'Commit'), - ('message', 'Commit message'), - ('user', 'Who'), - ('job-URL', 'Details') - ]) - details = ['> ' + l + ': ' + self.details[k] for k, l in labels.items() if k in self.details] - context = 'The latest commit contains errors. Please fix them. ' + '\n' + '\n'.join(details) - if "user-email" in self.details: - context = self.format_slack_mention(self.details["user-email"]) + ' ' + context + return logger - msgs = '\n\n'.join([context] + self.buffer) - self.post_to_slack(msgs) - self.buffer = [] +logger = create_logger() -fileConfig(log_ini_file, defaults={'logfile': log_file}) -logger = getLogger('sLogger') - - -def report_error(error, details=None, solutions=None): - """ - :param error: string - :param details: list of strings - :param solutions: list of strings - """ - # details - if details is not None: - details = ['> ' + s for s in details] - details = ['[DETAILS]:'] + details - details = '\n'.join(details) - # solutions - if solutions is not None: - solutions = ['> ' + s for s in solutions] - solutions = ['[SOLUTIONS]:'] + solutions - solutions = '\n'.join(solutions) - msg = '[ERROR]: ' + error - if details is not None: - msg += '\n' + details - if solutions is not None: - msg += '\n' + solutions - logger.error(msg) diff --git a/data/scratch/.gitignore b/data/scratch/.gitignore new file mode 100644 index 0000000..f4f9c93 --- /dev/null +++ b/data/scratch/.gitignore @@ -0,0 +1,2 @@ +/*.xls +!/reference.xls diff --git a/run.sh b/run.sh index 25f98cf..4a49122 100644 --- a/run.sh +++ b/run.sh @@ -6,6 +6,16 @@ # Convenient wrapper for running SportDP Helper BASEDIR="$( dirname "$0")" + +last="${@:$#}" # last parameter +other="${*%${!#}}" # all parameters except the last + +if [ -e "$last" ]; then + data="data/scratch/data.xls" + cp "$last" "$BASEDIR/$data" + last="./data/scratch/data.xls" +fi + cd "$BASEDIR" # enable display forwarding for selenium @@ -18,4 +28,5 @@ sudo docker run \ --name sportdb-helper-container \ -e DISPLAY=$DISPLAY \ -v /tmp/.X11-unix:/tmp/.X11-unix:ro \ - sportdb-helper "$@" \ No newline at end of file + -v $(pwd)/data:/sportdb-helper/data \ + sportdb-helper $last $other \ No newline at end of file