From aaa00bc2588d88dedd293dfe647ac4dcb99ac3cd Mon Sep 17 00:00:00 2001 From: Naoyuki Tai Date: Tue, 24 Sep 2024 12:06:56 -0400 Subject: [PATCH] Use alternate "sendmail" to talk to non-local MTA - mail.arxiv.org --- script/sync_prod_to_gcp/submissions_to_gcp.py | 43 +++++++++++++------ 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/script/sync_prod_to_gcp/submissions_to_gcp.py b/script/sync_prod_to_gcp/submissions_to_gcp.py index 3b8e13ce..ea2e9e06 100644 --- a/script/sync_prod_to_gcp/submissions_to_gcp.py +++ b/script/sync_prod_to_gcp/submissions_to_gcp.py @@ -343,7 +343,7 @@ class SubmissionFilesState: publish_type: str version: str # from the publish event log_extra: dict - submission_source: typing.Optional[dict] + submission_source: typing.Optional[str] published_versions: typing.List[dict] _top_levels: dict @@ -460,13 +460,18 @@ def update_metadata(self): abs_path = self.source_path(".abs") dates = [] try: - with open(abs_path, "r", encoding="utf-8") as abs_fd: - for line in abs_fd.readlines(): - sline = line.strip() - if sline == "": - break - if sline.startswith("Date"): - dates.append(sline) + with open(abs_path, "rb") as abs_fd: + abstract = abs_fd.read() + for line in abstract.split(b'\n'): + try: + sline = line.decode('utf-8') + if sline == "": + break + if sline.startswith("Date"): + dates.append(sline) + except UnicodeDecodeError: + # Don't care the non- uft-8 lines as date line is a generated text. + pass except FileNotFoundError as exc: msg = f"abs file {abs_path} does not exist" @@ -759,7 +764,7 @@ def submission_message_to_file_state(data: dict, log_extra: dict, ask_webnode: b source_mtime=file_state.get_submission_mtime()) except sync_published_to_gcp.WebnodeException as exc: - raise MissingGeneratedFile("Failed to generate %s", pdf_path) from exc + raise MissingGeneratedFile("Failed to generate %s" % pdf_path) from exc except ReadTimeout as exc: # This happens when failed to talk to webnode @@ -799,7 +804,7 @@ def submission_message_to_file_state(data: dict, log_extra: dict, ask_webnode: b file_state.register_files('html-files', html_files) except sync_published_to_gcp.WebnodeException as _exc: - raise MissingGeneratedFile("Failed to generate %s", html_path) from _exc + raise MissingGeneratedFile("Failed to generate %s" % html_path) from _exc except Exception as _exc: logger.warning("ensure_html: %s", file_state.vxid.ids, extra=log_extra, @@ -944,10 +949,22 @@ def submission_callback(message: Message) -> None: help_needed = os.environ.get("TEX_COMPILATION_RECIPIENT", "help@arxiv.org") subject = f"Uploading of {paper_id}v{version} failed" mail_body = f"Hello EUST,\nSubmission uploading for {paper_id}v{version} has failed. Please resolve the issue.\n\nThis message is generated by a bot on arxiv-sync.serverfarm.cornell.edu.\n" - cmd = ["/usr/bin/mail", "-r", "developers@arxiv.org", "-s", subject, help_needed] + SENDER = os.environ.get("SENDER", "developers@arxiv.org") + SMTP_SERVER = os.environ.get("SMTP_SERVER", "mail.arxiv.org") + SMTP_PORT = os.environ.get("SMTP_PORT", "25") + cmd = ["/usr/local/bin/arxiv-mail", + "-t", help_needed, + "-f", SENDER, + "-m", SMTP_SERVER, + "-p", SMTP_PORT, + "-s", subject, + ] + # using sendmail + # cmd = ["/usr/bin/mail", "-r", "developers@arxiv.org", "-s", subject, help_needed] + mail = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) try: - mail.communicate(mail_body, timeout=60) + mail.communicate(mail_body.encode('utf-8'), timeout=60) if mail.returncode == 0: # Once the email message is sent, done here. message.ack() @@ -983,7 +1000,7 @@ def submission_callback(message: Message) -> None: -def sync_to_gcp(state: SubmissionFilesState, log_extra: dict) -> bool: +def sync_to_gcp(state: SubmissionFilesState, log_extra: dict) -> None: """ From the submission file state, copy the ones that should bu uploaded.