+
+arXiv.org > {{ abs_meta.primary_archive.id }} > arXiv:{{ requested_id }}
+{{ config['BROWSE_SITE_LABEL'] }} > {{ abs_meta.primary_archive.id }} > arXiv:{{ requested_id }}
-
@@ -57,75 +60,30 @@ arXiv.org >
{% if abs_meta.primary_archive.id != abs_meta.primary_category.id %}{{ abs_meta.primary_archive.name }} > {% endif %}{{ abs_meta.primary_category.name }}
- Title:{{ abs_meta.title|tex_to_utf|arxiv_id_urls }}
-
- {{ generate_dateline() }}
-
- {# Below must equlvelent to what is done in the submission preview so the abstract appears as the author saw it in the preview. #}
- Abstract:{{ abs_meta.abstract|tex_to_utf_no_symb|arxiv_urlize|line_feed_to_br }}
-
-
+ {{ base_macros.abs(
+ abs_meta.arxiv_identifier.id,
+ abs_meta.title,
+ display_authors_with_links(abs_meta, author_links),
+ abs_meta.abstract,
+ abs_meta.get_datetime_of_version(abs_meta.version),
+ abs_meta.primary_category.id,
+ comments = abs_meta.comments,
+ msc_class = abs_meta.msc_class,
+ acm_class = abs_meta.acm_class,
+ journal_ref = abs_meta.journal_ref,
+ doi = abs_meta.doi,
+ report_num = abs_meta.report_num,
+ version = abs_meta.version,
+ submission_history = abs_meta.version_history,
+ secondary_categories = abs_meta.get_secondaries(),
+ include_stylesheet = 0) }}
+
- Submission history
From: {{ abs_meta.submitter.name|tex_to_utf if abs_meta.submitter.name != None }} [view email]
+ Submission history
From: {{ abs_meta.submitter.name|tex2utf if abs_meta.submitter.name != None }} [view email]
{#- Extra message for proxy sites (i.e. Proxy line has username and id) -#}
{#- TODO: revisit this. Logic for display follows classic but is strange; sometimes a proxy is just a person's name. -#}
- {% if abs_meta.proxy != None and abs_meta.proxy|wordcount > 1 %} [via {{ abs_meta.proxy.split()[0]|tex_to_utf|upper }} proxy]{% endif %}
+ {% if abs_meta.proxy != None and abs_meta.proxy|wordcount > 1 %} [via {{ abs_meta.proxy.split()[0]|tex2utf|upper }} proxy]{% endif %}
{%- for version_entry in abs_meta.version_history -%}
{{ generate_version_entry(version_entry, abs_meta.version) }}
@@ -133,14 +91,15 @@ Submission history
From: {{ abs_meta.submitter.name|tex_to_utf if abs_m
- Which authors of this paper are endorsers? | Disable MathJax (What is MathJax?)
- Browse v0.1 released 2018-10-22
-
-
- {#- The following supports the arXiv Labs Bibliographic Explorer project: https://labs.arxiv.org/projects/bibexplorer -#}
- {% if config['LABS_BIBEXPLORER_ENABLED'] %}
-
- {% endif %}
+ Which authors of this paper are endorsers? | Disable MathJax (What is MathJax?)
+ {% include "feedback_collector.html" %}
+
+
+ {#- The following supports the arXiv Labs Bibliographic Explorer project: https://labs.arxiv.org/projects/bibexplorer -#}
+ {% if config['LABS_BIBEXPLORER_ENABLED'] %}
+
+ {% endif %}
+
{% endblock content %}
@@ -177,16 +136,16 @@ Submission history
From: {{ abs_meta.submitter.name|tex_to_utf if abs_m
{%- macro generate_social_media_tags() -%}
-
-
+
+
-
+
-
+
{%- endmacro -%}
{%- macro category_line() -%}
- {{- abs_meta.primary_category.unalias().display_str() -}}
+ {{- abs_meta.primary_category.unalias().display -}}
{%- if abs_meta.secondary_categories -%}
{%- for category in abs_meta.display_secondaries() -%}
; {{ category }}
diff --git a/browse/templates/abs/author_links.html b/browse/templates/abs/author_links.html
index 0647ca70a..a4aaef15c 100644
--- a/browse/templates/abs/author_links.html
+++ b/browse/templates/abs/author_links.html
@@ -1,18 +1,20 @@
-{%- macro author_list( authors ) %}
+{%- macro author_list(abs_meta, authors) %}
{%- for part in authors -%}
{%- if part is string -%}
{{- part|arxiv_urlize -}}
{%- else -%}
- {{part[0]}}
+ {{ part[0] }}
{%- endif -%}
{% endfor %}
{%- endmacro -%}
-{{- author_list( author_links[0] ) }}
+{%- macro display_authors_with_links(abs_meta, author_links) %}
+
+{{- author_list(abs_meta, author_links[0]) }}
{%- if author_links[1] %}
{% if author_links[2] == 1 %}
- {{author_list(author_links[1])}}
+ {{ author_list(author_links[1]) }}
{% else %}
@@ -57,3 +59,5 @@
{%- endif -%}
{% endif -%}
+
+{%- endmacro -%}
diff --git a/browse/templates/archive/archive_list_all.html b/browse/templates/archive/archive_list_all.html
new file mode 100644
index 000000000..5d04f44ca
--- /dev/null
+++ b/browse/templates/archive/archive_list_all.html
@@ -0,0 +1,40 @@
+{%- extends "base.html" -%}
+
+{% block title %}Archives{% endblock %}
+
+{% block header_h1 %}
+{{ config['BROWSE_SITE_LABEL'] }} > archives
+{% endblock %}
+
+
+{%- block content %}
+ Archives
+
+ {% if bad_archive is not none %}
+ No archive '{{bad_archive}}' -- available archives are:
+ {% endif %}
+
+ Choose one of the following archives.
+
+
+ {% for archive in archives %}
+ {% set id = archive[0] %}
+ {% set name = archive[1] %}
+ - {{name}} ({{id}})
+ {% endfor %}
+
+
+ or from one of the following archives which no longer accept submissions
+ (most have been subsumed into the archives listed above)
+
+
+
+ {% for archive in defunct %}
+ {% set id = archive[0] %}
+ {% set name = archive[1] %}
+ {% set sub_to = archive[2] %}
+ - {{name}} ({{id}}, subsumed into {{sub_to}})
+ {% endfor %}
+
+
+{%- endblock %}
diff --git a/browse/templates/archive/single_archive.html b/browse/templates/archive/single_archive.html
new file mode 100644
index 000000000..9062122a8
--- /dev/null
+++ b/browse/templates/archive/single_archive.html
@@ -0,0 +1,111 @@
+{%- extends "base.html" -%}
+
+{% block head %}
+{{ super() }}
+
+{% endblock head %}
+
+{% block title %}{{archive.name}}{% endblock %}
+
+{% block header_h1 %}
+
+{{ config['BROWSE_SITE_LABEL'] }} > {{archive_id}}
+
+{% endblock %}
+
+
+{%- block content %}
+ {{archive.name}} (since {{archive.start_date.strftime('%B %Y')}})
+
+ {% if subsumed_id %}
+ The {{subsumed_id}} archive has been subsumed into
+ {{subsuming_category['name']}} ({{subsumed_by}}).
+ {{subsuming_category['name']}} ({{subsumed_by}}) is part of the {{archive['name']}} archive.
+ {% endif %}
+
+
+ For a specific paper, enter the identifier into the top right search box.
+
+
+ - Browse:
+
+ -
+ new
+ (most recent mailing, with abstracts)
+
+ - recent
+ (last 5 mailings)
+
+ - current month's
+ {{archive_id}} listings
+ - specific year/month:
+
+ {#
+
+
+
+
+ - Catch-up:
+ {# hard coded for legacy /catchup for now #}
+
+
+ - Search within the {{archive_id}} archive
+
+ - Article statistics by year:
+ {% for url, year in stats_by_year %}
+ {{year}}
+ {% endfor %}
+
+
+
+
+ {% if category_list|length > 1 %}
+ Categories within {{archive.name}}
+
+ {% for category in category_list %}
+ - {{category.id}} - {{category.name}}
+ (new,
+ recent,
+ current month)
+ {{category.description}}
+
+ {% endfor %}
+
+ {% endif %}
+{%- endblock %}
diff --git a/browse/templates/base.html b/browse/templates/base.html
index 0806f8c5d..ba593bb21 100644
--- a/browse/templates/base.html
+++ b/browse/templates/base.html
@@ -8,35 +8,29 @@
{# disable for now #}
- {%- if config['BROWSE_PIWIK_ENABLED'] %}
-
+ {%- if config['BROWSE_ANALYTICS_ENABLED'] %}
+
-
+
{% endif -%}
- {%- include "feedback_collector_js.html" -%}
- {% if config['BROWSE_USER_BANNER_ENABLED'] %}
-
+ {% if config['BROWSE_USER_BANNER_ENABLED'] and (config['BROWSE_USER_BANNER_START_DATE'] < request_datetime) and (config['BROWSE_USER_BANNER_END_DATE'] > request_datetime) %}
+
-
+
{% endif -%}
{%- endblock head -%}
@@ -45,8 +39,8 @@
{% if config['BROWSE_USER_BANNER_ENABLED'] %}
{%- include "user_banner.html" -%}
{% endif -%}
- {%- if config['BROWSE_PIWIK_ENABLED'] -%}
- 
+ {%- if config['BROWSE_ANALYTICS_ENABLED'] -%}
+ 
{% endif -%}
@@ -61,7 +55,7 @@
{% block header %}
{#- The ignore_me link is not meant to be visible to users; it is meant to catch robots/crawlers not respecting robots.txt. aria-hidden prevents screenreaders from being caught. -#}
- {% block header_h1 %}arXiv.org
{% endblock header_h1%}
+ {% block header_h1 %}{{ config['BROWSE_SITE_LABEL'] or 'arXiv.org' }}
{% endblock header_h1%}
{% block login_link %}{% endblock %}
If you have a disability and are having trouble accessing information on this website or need materials in an alternate format,
- contact web-accessibility@cornell.edu for assistance.
+ contact web-accessibility@cornell.edu for assistance.
diff --git a/browse/templates/cookies.html b/browse/templates/cookies.html
new file mode 100644
index 000000000..87c2bf2f1
--- /dev/null
+++ b/browse/templates/cookies.html
@@ -0,0 +1,81 @@
+{%- extends "base.html" -%}
+
+{% block title %}Your cookie configuration{% endblock %}
+
+{% block header_h1 %}{{ config['BROWSE_SITE_LABEL'] }} > cookies
{% endblock header_h1 %}
+
+{%- block content %}
+Your cookie configuration for arXiv
+
+You can use this form to set preferences that will apply to all arXiv.org sites for this
+and future sessions when access from the same machine. Cookies are snippets of information
+stored on your machine and returned to our site with each request you make (click
+show additional debugging information below to see what is stored).
+This facility will not work if you have disabled cookies or on browsers that do not
+support cookies.
+
+For information about the full-text formats available, consult the
+viewing help.
+If the cookies are set successfully, then abstract and listing
+pages will automatically link to the appropriate versions of each
+paper. You may have to reload previously-visited abstract/listing
+pages to update the format links (Shift+Reload in many browsers), because
+the old versions of the pages will be cached by your browser.
+
+
+
+
+
+
+{% if debug %}
+Debugging information: dump of current cookie data
+
+ Name Value
+ {% for cooname in request.cookies.keys() %}
+
+ {% if cooname in controlled_cookies %}
+ {{cooname}}{{cooname}}{{ request.cookies[cooname] }}
+ {% endfor %}
+
+
+Only those cookies shown emphasized in the table above are controlled
+by this interface. arXiv uses other cookies such as browser and
+tapir_permanent to maintain session and permanent login settings.
+You can, if you want, clear all cookies using the controls in your browser.
+
+
+{% else %}
+(show additional debugging information)
+{% endif %}
+
+About cookies
+"HTTP Cookies" allow clients to retain certain state
+information to be included in future requests to a particular
+server. Cookies are now defined
+by RFC 6265. It is
+not necessary to use/permit cookies to read articles on arXiv.org.
+{% endblock content %}
diff --git a/browse/templates/feedback_collector.html b/browse/templates/feedback_collector.html
new file mode 100644
index 000000000..71143e7bd
--- /dev/null
+++ b/browse/templates/feedback_collector.html
@@ -0,0 +1,2 @@
+Browse v0.2.1 released 2019-04-18
+
diff --git a/browse/templates/feedback_collector_js.html b/browse/templates/feedback_collector_js.html
index 885aabc79..b0c05510b 100644
--- a/browse/templates/feedback_collector_js.html
+++ b/browse/templates/feedback_collector_js.html
@@ -8,8 +8,8 @@
});
},
fieldValues: {
- "components": ["15700"], // Browse component.
- "versions": ["14132"], // Release browse-0.1
+ "components": ["15700"], // Jira ID for browse component
+ "versions": ["14153"], // Jira ID for browse-0.2.1 release
"customfield_11401": window.location.href
}
};
diff --git a/browse/templates/home/home.html b/browse/templates/home/home.html
new file mode 100644
index 000000000..66961972d
--- /dev/null
+++ b/browse/templates/home/home.html
@@ -0,0 +1,95 @@
+{%- extends "base.html" -%}
+{% block title %}arXiv.org e-Print archive{% endblock %}
+{% block head %}
+ {{ super() -}}
+{% endblock head %}
+
+{% block login_link %}{% include 'login.html' %}{% endblock %}
+{% block body_id %}id="front"{% endblock %}
+
+{%- block content %}
+{#- TODO: display order in taxonomy? -#}
+Open access to {% if document_count -%}{{ "{:,}".format(document_count) }}{%- endif %}
+ e-prints in the fields of physics, mathematics, computer science, quantitative biology, quantitative finance, statistics, electrical engineering and systems science, and economics. Submissions to arXiv should conform to Cornell University academic standards. arXiv is owned and operated by Cornell University, a private not-for-profit educational institution. arXiv is funded by Cornell University, the Simons Foundation and by the member institutions.
+
+{#- /multi sends to either search, catchup or form interface based on which button is hit. -#}
+
+
+
+
+{%- include "home/news.html" -%}
+See cumulative "What's New" pages.
+Read robots beware before attempting any automated download
+
+{#- TODO: define display order in taxonomy? -#}
+{{- group_section(('grp_physics','grp_math','grp_cs','grp_q-bio','grp_q-fin','grp_stat','grp_eess','grp_econ')) }}
+
+
+{#- TODO: remove because of new footer? -#}
+About arXiv
+
+ - General information and Scientific Advisory Board
+ - Support and Governance Model and Member Advisory Board
+ - Find, view, email alerts and RSS feeds
+ - Submission and moderation details
+ - Usage statistics and news
+ - See also searchable help pages
+
+
+{% endblock content -%}
+
+{%- macro group_section(group_keys) -%}
+ {%- for group_key in group_keys -%}
+ {{ groups[group_key].name }}
+
+ {% for archive_key, archive_details in archives.items()|sort(attribute='1.name') if archives[archive_key].in_group == group_key %}
+ {%- set archive_search_url = url_for('search_archive', archive=archive_key) -%}
+ {%- set archive_url = url_for('browse.archive', archive=archive_key) if archive_key != 'cs' else 'https://arxiv.org/corr' -%}
+ {%- set archive_name = archives[archive_key].name if archive_key != 'cs' else 'Computing Research Repository' -%}
+
+ {#- TODO: url_for /list, /catchup -#}
+ -
+ {{ archive_name }} ({{ archive_key if archive_key != 'cs' else 'CoRR' }} new, recent, search{% if 0 %}, last month{% endif %})
+ {%- if group_key == 'grp_physics' and archive_key not in categories -%}
includes:
+ {%- elif archive_key == 'cs' -%}
includes (see detailed description):
+ {%- elif archive_key in ('eess', 'econ') -%}
includes (see detailed description):
+ {%- elif group_key != 'grp_physics' -%}
includes (see detailed description):
+ {%- endif -%}
+ {% for category_key, category_details in categories.items()|sort(attribute='1.name') if categories[category_key].in_archive == archive_key %}
+ {% if not (loop.first and loop.last and group_key == 'grp_physics') %}{{ categories[category_key].name }}{% if not loop.last %}; {% endif %}{% endif %}
+ {% endfor %}
+
+ {% endfor %}
+
+ {% endfor %}
+{%- endmacro -%}
diff --git a/browse/templates/home/news.html b/browse/templates/home/news.html
new file mode 100644
index 000000000..6fd2773fb
--- /dev/null
+++ b/browse/templates/home/news.html
@@ -0,0 +1,3 @@
+{#- News blurbs appear at the top of the home page. Generally there should be no more than four items. -#}
+14 Jan 2019: The annual update from the arXiv team is now available
+5 Sept 2018: arXiv looks to the future with move to Cornell CIS
diff --git a/browse/templates/list/base.html b/browse/templates/list/base.html
new file mode 100644
index 000000000..6f3673342
--- /dev/null
+++ b/browse/templates/list/base.html
@@ -0,0 +1,155 @@
+{%- extends "base.html" -%}
+
+{% block title %}{{list_ctx_name}} {{list_month_name}} {{list_year}}{% endblock %}
+
+{% block header_h1 %}
+
+ {{ config['BROWSE_SITE_LABEL'] }} >
+ {{list_ctx_id}}
+
+{% endblock %}
+
+{%- block content %}
+
+
+{%- endblock %}
+
+
+{%- macro page_divs() -%}
+[ total of {{count}} entries
+ {{ pages() }}
+ ]
+
+[ showing {{shown}} entries per page:
+ {{ show_link('fewer',mf_fewer) }} |
+ {{ show_link('more', mf_more) }} |
+ {{ show_link('all', mf_all) }} ]
+
+{%- endmacro -%}
+
+
+{% macro item_list(items, heading) %}
+
+ {% if heading %}
+ {{heading}}
+ {% endif %}
+
+ {% for item in items %}
+ {% set article = item['article'] %}
+ {% set ids = article.arxiv_identifier.ids %}
+ {% set listing_index = item['list_index'] %}
+ -
+ [{{listing_index}}]
+
+ arXiv:{{ article.arxiv_identifier.ids }}
+
+ {{ type_info( item ) }}
+ {{ dl_links( article ) }}
+
+ -
+
+
+ {% endfor %}
+
+{% endmacro %}
+
+
+{%- macro do_author_list(article) -%}
+ {% set au_list_tup = author_links[article.arxiv_id_v] %}
+ {%- if not au_list_tup or not au_list_tup[0] -%}
+ (author list is unavailable)
+ {%- else -%}
+ {%- for part in au_list_tup[0] -%}
+ {%- if part is string -%}
+ {{- part|arxiv_urlize -}}
+ {%- else -%}
+ {{part[0]}}
+ {%- endif -%}
+ {% endfor %}
+ {%- endif -%}
+{%- endmacro -%}
+
+
+{%- macro pages() -%}
+{% if pg|length != 0 or not viewing_all %}
+:
+ {%- for pg in paging -%}
+ {% if pg.nolink is defined %}
+ {{pg.nolink}}
+ {% else %}
+ {{pg.txt}}
+ {% endif %}
+ {% endfor %}
+{% endif %}
+{%- endmacro -%}
+
+
+{%- macro show_link( txt, to_show ) %}
+ {% if to_show %}
+
+ {{txt}}
+ {% else %}
+ {{txt}}
+ {% endif %}
+{%- endmacro -%}
+
+{%- macro dl_links( article ) -%}
+ {% set id=article.arxiv_id %}
+ {% set downloads = downloads[ article.arxiv_id_v ] %}
+ [pdf
+ {%- if 'ps' in downloads -%}
+ , ps
+ {%- endif -%}
+ {%- if 'other' in downloads -%}
+ , other
+ {%- endif -%}
+ ]
+{%- endmacro -%}
+
+{% macro type_info( item ) %}
+ {% if item['listingType'] == 'cross' %}
+ (cross-list from {{item['primary']}})
+ {% endif %}
+ {% if item['listingType'] == 'rep' %}
+ (replaced)
+ {% endif %}
+{% endmacro %}
diff --git a/browse/templates/list/month.html b/browse/templates/list/month.html
new file mode 100644
index 000000000..fcfda4aad
--- /dev/null
+++ b/browse/templates/list/month.html
@@ -0,0 +1,5 @@
+{%- extends "list/base.html" -%}
+
+{% block list_index %}
+Authors and titles for {{pubmonth.strftime('%B %Y')}}
+{% endblock %}
diff --git a/browse/templates/list/new.html b/browse/templates/list/new.html
new file mode 100644
index 000000000..18b133599
--- /dev/null
+++ b/browse/templates/list/new.html
@@ -0,0 +1,38 @@
+{%- extends "list/base.html" -%}
+
+{#
+New is quite different than the other list pages.
+It has an index of types of items.
+It needs to display the submitted dates and announced dates.
+It needs to do headings for the new/cross/rep sections.
+#}
+
+{% block list_index %}
+{{ listdate() }}
+{{ type_index() }}
+{% endblock %}
+
+{% block items %}
+ {% for sec in sub_sections_for_types %}
+ {{ item_list(sec['items'], sec['heading']) }}
+ {% endfor %}
+{% endblock %}
+
+{% macro type_index() %}
+ {% if index_for_types %}
+
+ {% for ityp in index_for_types %}
+ - {{ityp[0]}}
+ {% endfor %}
+
+{% endif %}
+{% endmacro %}
+
+{% macro listdate() %}
+ {% if announced %}
+ Submissions received from
+ {{ submitted[0].strftime('%A, %-d %B %Y') }} to {{ submitted[1].strftime('%A, %-d %B %Y') }},
+ announced {{ announced.strftime('%A, %-d %B %Y') }}
+
+ {% endif %}
+{% endmacro %}
diff --git a/browse/templates/list/recent.html b/browse/templates/list/recent.html
new file mode 100644
index 000000000..7e9c578ea
--- /dev/null
+++ b/browse/templates/list/recent.html
@@ -0,0 +1,24 @@
+{%- extends "list/base.html" -%}
+
+{% block list_index %}
+Authors and titles for recent submissions
+
+
+ {% for pdate in pubdates %}
+ {% set skipfordate = pdate[1] %}
+ -
+
+ {{ pdate[0].strftime('%A, %-d %B %Y') }}
+
+
+ {% endfor %}
+
+{% endblock %}
+
+
+{% macro listdate() %}
+ {% if announced %}
+ Submissions announced {{ announced.strftime('%A, %-d %B %Y') }}
+
+ {% endif %}
+{% endmacro %}
diff --git a/browse/templates/list/year.html b/browse/templates/list/year.html
new file mode 100644
index 000000000..8391a4dbe
--- /dev/null
+++ b/browse/templates/list/year.html
@@ -0,0 +1,5 @@
+{%- extends "list/base.html" -%}
+
+{% block list_index %}
+Authors and titles for {{pubmonth.strftime('%Y')}}
+{% endblock %}
diff --git a/browse/templates/login.html b/browse/templates/login.html
new file mode 100644
index 000000000..a3cbf2e76
--- /dev/null
+++ b/browse/templates/login.html
@@ -0,0 +1,7 @@
+
+{% if request.auth.user is defined %}
+ Logged in as {{ request.auth.user.username }} | My Account | Logout
+{% else %}
+ Login
+{% endif %}
+
diff --git a/browse/templates/stats/base.html b/browse/templates/stats/base.html
new file mode 100644
index 000000000..f36e88a98
--- /dev/null
+++ b/browse/templates/stats/base.html
@@ -0,0 +1,14 @@
+{%- extends "base.html" -%}
+{# Do not show login status on stats pages for now, per classic #}
+
+{% block head %}
+ {{ super() -}}
+ {# TODO: upgrade d3 to v5.x? #}
+
+{% endblock head %}
+
+{% block header_h1 %}{{ config['BROWSE_SITE_LABEL'] }} > stats > server usage
{% endblock %}
+
+{%- block content %}
+See also other arXiv usage statistics.
+{% endblock content %}
diff --git a/browse/templates/stats/monthly_downloads.html b/browse/templates/stats/monthly_downloads.html
new file mode 100644
index 000000000..222fd5ec9
--- /dev/null
+++ b/browse/templates/stats/monthly_downloads.html
@@ -0,0 +1,21 @@
+{%- extends "stats/base.html" -%}
+
+{% block title %}Monthly Download Rates{% endblock %}
+{% block header_h1 %}{{ config['BROWSE_SITE_LABEL'] }} > stats > monthly downloads
{% endblock %}
+
+{% block content %}
+arXiv Monthly Download Rates [CSV]
+
+{% include "stats/monthly_downloads_js.html" %}
+
+Blue: Number of downloads per month.
Click and drag in the small graph below to pan or zoom.
+Total number of downloads {% if most_recent_dt %}through {{ most_recent_dt.strftime('%B %Y') }}{% endif %} = {{ "{:,}".format(total_downloads) }}
+
+ Caveats: While we have attempted to extract download data representing unique full-text downloads by real users, there are many factors which affect accuracy. These factors include: 1) the data is from the main arXiv site and the
+ arXiv mirrors, though some mirror data is incomplete; 2) only web downloads are included (and not FTP or email "downloads" that were formerly supported); 3) we have counted downloads according to the COUNTER algorithm which excludes
+ rapid repeat downloads; 4) we have attempted to identify and remove robot or automated downloads from the count (false positives lead to undercounting, failing to identify robots leads to overcounting); 5) data prior to 2009 has not been cleaned
+ with as much care as later data, it shows trends nonetheless.
+
+
+{{ super() }}
+{% endblock content %}
diff --git a/browse/templates/stats/monthly_downloads_js.html b/browse/templates/stats/monthly_downloads_js.html
new file mode 100644
index 000000000..84833c98d
--- /dev/null
+++ b/browse/templates/stats/monthly_downloads_js.html
@@ -0,0 +1,160 @@
+
+
diff --git a/browse/templates/stats/monthly_submissions.html b/browse/templates/stats/monthly_submissions.html
new file mode 100644
index 000000000..216f26fb8
--- /dev/null
+++ b/browse/templates/stats/monthly_submissions.html
@@ -0,0 +1,16 @@
+{%- extends "stats/base.html" -%}
+
+{% block title %}Monthly Submissions{% endblock %}
+{% block header_h1 %}{{ config['BROWSE_SITE_LABEL'] }} > stats > monthly submission rates
{% endblock %}
+
+{% block content %}
+arXiv Monthly Submission Rates [CSV]
+{% include "stats/monthly_submissions_js.html" %}
+Blue: Number of new submissions received during each month since {{ arxiv_start_dt.strftime('%B %Y') }}.
+ Hover over the graph to see the exact count for a given month.
+Total number of submissions shown in graph as of {{ current_dt.strftime('%B %-d, %Y') }} (after {{ "%.1f"|format(arxiv_age_years|float) }} years) = {{ "{:,}".format(num_submissions) }}
+
+ The total number of submissions excludes {{ "{:,}".format(num_migrated) }} articles that were migrated to arXiv rather than being submitted directly, and includes {{ "{:,}".format(num_deleted) }} articles that have been deleted. The total number of articles available is {{ "{:,}".format(num_submissions_adjusted) }}.
+
+{{ super() }}
+{% endblock content %}
diff --git a/browse/templates/stats/monthly_submissions_js.html b/browse/templates/stats/monthly_submissions_js.html
new file mode 100644
index 000000000..b8f9bd946
--- /dev/null
+++ b/browse/templates/stats/monthly_submissions_js.html
@@ -0,0 +1,199 @@
+
+
+
diff --git a/browse/templates/stats/today.html b/browse/templates/stats/today.html
new file mode 100644
index 000000000..2309d99a0
--- /dev/null
+++ b/browse/templates/stats/today.html
@@ -0,0 +1,16 @@
+{%- extends "stats/base.html" -%}
+
+{% block title %}hourly usage{% endblock %}
+
+{% block content %}
+
+arXiv Web Server Usage from {{ requested_dt.strftime('%A, %B %-d, %Y') }} ({{ config['BROWSE_SITE_LABEL'] }} site only) [CSV]
+{% if normal_count > 0 %}
+{% include "stats/today_js.html" %}
+Total number of connections = {{ "{:,}".format(normal_count) }} (+{{ "{:,}".format(admin_count) }} local & administrative connections)
+{% else %}
+
No data currently available.
+{% endif %}
+
Current local time is {{ current_dt.strftime('%A, %B, %-d, %Y %H:%M%:%S') }} US/Eastern
+{{ super() }}
+{% endblock content %}
diff --git a/browse/templates/stats/today_js.html b/browse/templates/stats/today_js.html
new file mode 100644
index 000000000..0cbc2c60f
--- /dev/null
+++ b/browse/templates/stats/today_js.html
@@ -0,0 +1,125 @@
+
+
diff --git a/browse/templates/tb/404.html b/browse/templates/tb/404.html
new file mode 100644
index 000000000..62d990fa8
--- /dev/null
+++ b/browse/templates/tb/404.html
@@ -0,0 +1,32 @@
+{%- extends "tb/base.html" -%}
+
+{% block title %}
+{% if arxiv_id %}
+[{{ arxiv_id }}] Article identifier not {% if not_found %}found{% else %}recognized{% endif %}
+{% elif missing_id %}
+No article ID specified
+{% else %}
+Trackback not found
+{% endif %}
+{% endblock %}
+
+{%- block content %}
+{% if arxiv_id or missing_id %}
+ {% if arxiv_id and not_found %}
+ Article 'arXiv:{{ arxiv_id }}' not found
+ The identifier you have specified (arXiv:{{ arxiv_id }}) may be invalid. Please inform help@arxiv.org if you believe that the identifier should correspond to a valid paper in arXiv.
+ {% elif arxiv_id %}
+ Article identifier '{{ arxiv_id }}' not recognized
+ {% else %}
+ No article identifier specified
+ {% endif %}
+ We were unable to extract an arXiv article ID from the URL specified. You may:
+{% else %}
+ Trackback not found
+ Badly formed redirect request. If the link you followed was on arXiv, please report this error to us. If it was elsewhere, we suggest you report it to the maintainer of that site.
+{% endif %}
+
+ - View recent trackbacks
+ - View the trackbacks for a particular article at
https://arxiv.org{{ url_for('.tb', arxiv_id='') }}{arXiv_id}
+
+{% endblock content %}
diff --git a/browse/templates/tb/base.html b/browse/templates/tb/base.html
new file mode 100644
index 000000000..e86a302b5
--- /dev/null
+++ b/browse/templates/tb/base.html
@@ -0,0 +1,27 @@
+{%- extends "base.html" -%}
+{# Do not show login status on these pages for now, per classic #}
+
+{% block head %}
+ {{ super() -}}
+
+
+{% endblock head %}
+
+{%- block content %}
+
+ About trackbacks
+ By sending a trackback, you can notify
+ arXiv.org that you have created a web page that references a paper. Popular
+ blogging software supports trackback: you can send us a trackback about this
+ paper by giving your software the following trackback URL:
+ https://arxiv.org/trackback/{arXiv_id}
+ Some blogging software supports trackback autodiscovery -- in this
+ case, your software will automatically send a trackback as soon as your
+ create a link to our abstract page. See our
+ trackback help page for more information.
+
+
+{% endblock content %}
diff --git a/browse/templates/tb/macros.html b/browse/templates/tb/macros.html
new file mode 100644
index 000000000..4b93e4531
--- /dev/null
+++ b/browse/templates/tb/macros.html
@@ -0,0 +1,7 @@
+{%- macro generate_trackback_link(trackback, include_posted_date=True) -%}
+
+ {{- trackback.title|entity_to_utf|truncate(150) }} [
+ {%- if trackback.blog_name %}{{ trackback.blog_name|trim|entity_to_utf|truncate(50) -}} @ {% endif -%}
+ {%- if trackback.has_valid_url %}{{ trackback.display_url|truncate(30,False) }}{% else %}INVALID-URL{% endif -%}]
+ {%- if include_posted_date %} [trackback posted {{ trackback.posted_datetime.strftime('%a, %-d %b %Y %H:%M:%S %Z') }}]{% endif %}
+{%- endmacro -%}
diff --git a/browse/templates/tb/recent.html b/browse/templates/tb/recent.html
new file mode 100644
index 000000000..efffeb707
--- /dev/null
+++ b/browse/templates/tb/recent.html
@@ -0,0 +1,42 @@
+{%- extends "tb/base.html" -%}
+{%- from 'tb/macros.html' import generate_trackback_link -%}
+{%- import 'base/macros.html' as base_macros -%}
+
+{% block title %}Recent Trackbacks{% endblock %}
+{% block header_h1 %}{{ config['BROWSE_SITE_LABEL'] }} > recent trackbacks
{% endblock %}
+
+{%- block content %}
+ Recent Trackbacks
+ Trackbacks indicate external web sites that link to articles in arXiv.org. Trackbacks may not reflect the opinion of arXiv.org or of that article's authors.
+
+ {% if recent_trackback_pings -%}
+ {%- set ns = namespace(cur_tb_ymd='', cur_tb_url='') -%}
+ {%- for recent_tb in recent_trackback_pings -%}
+ {%- set this_tb_ymd = recent_tb[0].posted_datetime.strftime('%B %-d, %Y') -%}
+ {%- set this_tb_url = recent_tb[0].url -%}
+ {%- if this_tb_ymd != ns.cur_tb_ymd -%}
+ {%- set ns.cur_tb_ymd = this_tb_ymd -%}
+ {{ this_tb_ymd }}
+ {%- endif -%}
+ {%- if this_tb_url != ns.cur_tb_url -%}
+ {%- set ns.cur_tb_url = this_tb_url -%}
+ {{- generate_trackback_link(recent_tb[0], include_posted_date=False) }} links the following articles:
+
+ {%- for article in article_map[this_tb_url] -%}
+ - {{ article[1]|tex2utf }} [{{ article[0] }}]
+ {%- endfor -%}
+
+ {%- endif -%}
+ {%- endfor -%}
+ {%- endif %}
+ {{ super() }}
+{% endblock content %}
diff --git a/browse/templates/tb/tb.html b/browse/templates/tb/tb.html
new file mode 100644
index 000000000..ab02b8d9f
--- /dev/null
+++ b/browse/templates/tb/tb.html
@@ -0,0 +1,42 @@
+{%- extends "tb/base.html" -%}
+{%- from 'abs/author_links.html' import display_authors_with_links -%}
+{%- import 'base/macros.html' as base_macros -%}
+{%- from 'tb/macros.html' import generate_trackback_link -%}
+
+{% block title %}Article Trackbacks{% endblock %}
+{% block header_h1 %}{{ config['BROWSE_SITE_LABEL'] }} > article trackbacks
{% endblock %}
+{# Disable login on this page for now, per classic #}
+{% block login_link %}{% endblock %}
+
+{%- block content %}
+ {% if trackback_pings %}
+ Trackbacks for {{ arxiv_identifier.id }}
+ {% for tb in trackback_pings %}
+ {{- generate_trackback_link(tb) -}}
+ {% endfor %}
+
+ {#- abstract field is deliberately suppressed in call to abs macro. -#}
+ {%- if abs_meta -%}
+ {{ base_macros.abs(
+ abs_meta.arxiv_identifier.id,
+ abs_meta.title,
+ display_authors_with_links(abs_meta, author_links),
+ '',
+ abs_meta.get_datetime_of_version(abs_meta.version),
+ abs_meta.primary_category.id,
+ comments = abs_meta.comments,
+ msc_class = abs_meta.msc_class,
+ acm_class = abs_meta.acm_class,
+ journal_ref = abs_meta.journal_ref,
+ doi = abs_meta.doi,
+ report_num = abs_meta.report_num,
+ version = abs_meta.version,
+ submission_history = abs_meta.version_history,
+ secondary_categories = abs_meta.get_secondaries()) }}
+
+ {%- endif -%}
+ {% else %}
+ There are no trackback pings recorded for {{ arxiv_identifier.id }}.
+ {% endif %}
+ {{ super() }}
+{% endblock content %}
diff --git a/browse/templates/user_banner.html b/browse/templates/user_banner.html
index 19d33ec10..8ec524811 100644
--- a/browse/templates/user_banner.html
+++ b/browse/templates/user_banner.html
@@ -1,4 +1,19 @@
-
-Attention Readers: arXiv will be unavailable due to maintenance ~20 minutes starting 05:00 ET (09:00 UTC) on Thursday, November 1, 2018.
-
+
diff --git a/browse/templates/year.html b/browse/templates/year.html
new file mode 100644
index 000000000..d3c205c10
--- /dev/null
+++ b/browse/templates/year.html
@@ -0,0 +1,54 @@
+{% extends "base.html" %}
+
+{% block title %}{{archive.name}}{% endblock %}
+
+
+{% block header_h1 %}
+
+{{ config['BROWSE_SITE_LABEL'] }} > {{archive_id}} > {{year}}
+
+{% endblock %}
+
+
+{% block content %}
+{{archive.name}}
+Article statistics for {{year}}
+Available montly lists with counts of {{archive_id}} articles +
+cross-listings to {{archive_id}} in {{year}}
+(each '|' represents 20 articles):
+
+TODO fake data as of 2019-03-04
+
+
+ {% for month in months %}
+ -
+ {{month['yymm']}}
+
+ {% for txt, url in month['art'] -%}
+ {%- if url -%}
+ {{txt}}
+ {%- else -%}
+ {{txt}}
+ {%- endif -%}
+ {%- endfor %}
+ {{month['new']|default('0',true)}} + {{month['cross']|default('0',true)}}
+
+ {% endfor %}
+
+
+{{year}} totals: {{listing['new_count']|default('unknown')}} articles + {{listing['cross_count']|default('unknown')}} cross-lists
+
+Other years:
+
Article statistics by year:
+ {% for url, year in stats_by_year %}
+ {% if url %}
+ {{year}}
+ {% else %}
+ {{year}}
+ {% endif %}
+ {% endfor %}
+
+
+
+
+{% endblock %}
diff --git a/browse/util/__init__.py b/browse/util/__init__.py
deleted file mode 100644
index a7a09d0fa..000000000
--- a/browse/util/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-"""Utility code for browse."""
diff --git a/browse/util/clickthrough.py b/browse/util/clickthrough.py
deleted file mode 100644
index 2e8b849cb..000000000
--- a/browse/util/clickthrough.py
+++ /dev/null
@@ -1,30 +0,0 @@
-"""
-Functions to create hashes for clickthrough links.
-
-These are used to log to the web acccess logs when the DOI and
-bookmarking links are clicked.
-
-The hash is used to prevent malicious use of the click through
-controller to create links that look like they are on arXiv but get
-redirected someplace undesirable.
-"""
-
-from typing import Callable
-import hashlib
-
-
-def create_hash(secret: str, url: str) -> str:
- """Create a hash of the secret and url."""
- s = f'{secret}{url}'
- return str(hashlib.md5(s.encode()).hexdigest()[0:8])
-
-
-def is_hash_valid(secret: str, url: str, ct_hash: str) -> bool:
- """Check that ct_hash was generated by create_hash for secret and url."""
- return ct_hash == create_hash(secret, url)
-
-
-def create_ct_url(secret: str, url_for: Callable[..., str], url: str) -> str:
- """Create a URL to the clickthrough service with a valid hash."""
- url_for('browse.clickthrough', url=url, v=create_hash(secret, url))
- return url_for('browse.clickthrough', url=url, v=create_hash(secret, url))
diff --git a/browse/util/id_patterns.py b/browse/util/id_patterns.py
deleted file mode 100644
index 325242fa7..000000000
--- a/browse/util/id_patterns.py
+++ /dev/null
@@ -1,329 +0,0 @@
-r"""Patterns and functions to detect arXiv ids and Urls in text.
-
-Functions to detech arXiv ids, URLs and DOI in text.
-Functions to transform them to tags.
-
-These were originally jinja filters but became a little too big
-for that so they were split out and made more general so they didn't
-rely on the Flask context.
-
-These all use expect input of Markup or non-markup text and return
-Markup objects. This is because the that get added need to avoid
-double escaping.
-
-There are several classes of patterns we want to match but there is
-some overlap in these patterns. To avoid looking for and parsing HTML in each
-jinja filter, detecting these patterns is combined.
-
-So far we are looking for:
-DOIs DOI: 10.1145/0001234.1234567
-arXiv IDS: 1234.12345 1234.12345v1 hep-ph1307.1843
-HTTP URLs: http://something.org/myPaper/1234.12345
-FTP URLs: ftp://example.com/files/1234.12345
-
-Just matching for arXiv ids with \d{4}\.\d{4,5} will match several of
-these. To deal with this we are priortizing the matches and
-interupting once one is found.
-
-We should probably match DOIs first because they are the source of a
-lot of false positives for arxiv matches.
-"""
-from typing import Optional, List, Pattern, Match, Tuple, Callable
-import re
-from dataclasses import dataclass
-
-from urllib.parse import quote
-from jinja2 import Markup, escape
-
-from arxiv import taxonomy
-
-
-@dataclass
-class Matchable:
- """Class for paterns."""
-
- examples: List[str]
- pattern: Pattern
-
-
-def _identity(x: str)->str:
- """Identity funciton for default in some places."""
- return x
-
-
-doi_patterns = [
- Matchable(['10.1145/0001234.1234567'],
- re.compile(r'(?P10.\d{4,9}/[-._;()/:A-Z0-9]+)', re.I))
-]
-"""List of Matchable for DOIs in text.
-
-We should probably match DOIs first because they are the source of a
-lot of false positives for arxiv matches.
-
-Only using the most general express from
-https://www.crossref.org/blog/dois-and-matching-regular-expressions/
-"""
-
-_archive = '|'.join([re.escape(key) for key in taxonomy.definitions.ARCHIVES.keys()])
-"""string for use in Regex for all arXiv archives"""
-
-_category = '|'.join([re.escape(key) for key in taxonomy.definitions.CATEGORIES.keys()])
-
-_arxiv_id_prefix = r'(?Par[xX]iv:)?'
-"""Attempt to catch the arxiv prefix in front of arxiv ids so it can be
-included in the tag anchor. ARXIVNG-1284"""
-
-basic_arxiv_id_patterns = [
- Matchable(['math/0501233', 'hep-ph/0611734', 'gr-qc/0112123'],
- re.compile(_arxiv_id_prefix + r'(?P(%s)\/\d{2}[01]\d{4}(v\d*)?)'
- % _archive, re.I)),
- Matchable(['1609.05068', '1207.1234v1', '1207.1234', '1807.12345',
- '1807.12345v1', '1807.12345v12'],
- re.compile(r'(?\d{4}\.\d{4,5}(v\d*)?)',
- re.I)),
- Matchable(['math.GR/0601136v3', 'math.GR/0601136'],
- re.compile(_arxiv_id_prefix + r'(?P(%s)\/\d{2}[01]\d{4}(v\d*)?)'
- % _category, re.I))
-]
-
-OKCHARS = r'([a-z0-9,_.\-+~:]|%[a-f0-9]*)'
-"""Chacters that are acceptable during PATH, QUERY and ANCHOR parts"""
-
-HOST_NAME = r'(?:[a-z0-9][a-z0-9\-.:]+[a-z0-9])'
-"""Regex used to match host names in arXiv urlize.
-
-This is not a perfect regex for a host name, It accepts only a sub-set
-of hostnames to meet the needs of arxiv.
-
-HOST_NAME must end with a simplified character to avoid capturing a
-period.
-"""
-
-PATH = rf'(?P(/{OKCHARS}*)+)?'
-"""Regex for path part of URLs for use in urlize"""
-
-QUERY = rf'(?P\?(&?({OKCHARS}*(={OKCHARS}*)?))*)?'
-"""Regex for query part of URLs for use in urlize"""
-
-ANCHOR = rf'(?P#({OKCHARS}|/)*)?'
-"""Regex for anchor part of URLs for use in urlize"""
-
-URLINTEXT_PAT = re.compile(r'(?P(?:https?://)'
- f'{HOST_NAME}{PATH}{QUERY}{ANCHOR})',
- re.I)
-"""Regex to match URLs in text."""
-
-FTP_PAT = re.compile(rf'(?P(?:ftp://)({OKCHARS}|(@))*{PATH})', re.I)
-"""Regex to match FTP URLs in text."""
-
-basic_url_patterns = [
- Matchable(['http://something.com/bla'], URLINTEXT_PAT),
- Matchable(['ftp://something.com/bla'], FTP_PAT)
-]
-"""List of Matchable to use when finding URLs in text"""
-
-bad_arxiv_id_patterns = [
- re.compile('vixra', re.I), # don't need to link to vixra
-]
-"""List of Regex patterns that will cause matching to be skipped for
-the token."""
-
-dois_ids_and_urls = basic_url_patterns + doi_patterns + basic_arxiv_id_patterns
-"""List of Matchable to use when finding DOIs, arXiv IDs, and URLs.
-
-URLs are first because some URLs contain DOIs or arXiv IDS.
-
-DOI are before arXiv ids because many DOIs are falsely matched by the
-arxiv_id patterns.
-"""
-
-
-_bad_endings = ['.', ',', ':', ';', '&', '(', '[', '{']
-"""These should not appear at the end of URLs because they are likely
-part of the surrounding text"""
-
-
-def _find_match(patterns: List[Matchable], token: str) \
- -> Optional[Tuple[Match, Matchable]]:
- """Find first in patterns that is found in txt."""
- for chgMtch in patterns:
- if chgMtch.pattern.flags:
- fnd = re.search(chgMtch.pattern, token)
- else:
- fnd = re.search(chgMtch.pattern, token, re.I)
- if fnd is not None:
- return (fnd, chgMtch)
- return None
-
-
-def _transform_token(patterns: List[Matchable],
- bad_patterns: List[Pattern],
- id_to_url: Callable[[str], str],
- doi_to_url: Callable[[str], str],
- url_to_url: Callable[[str], str],
- token: str) -> str:
- """
- Transform a token from text to one of the Matchables.
-
- This only transforms against the first of Matchable matched.
- Matching on this token will be skipped if any of the bad_patterns
- match the token (that is re.search).
- """
- id_to_url = id_to_url or (lambda x: x)
- doi_to_url = doi_to_url or (lambda x: x)
- url_to_url = url_to_url or (lambda x: x)
-
- for pattern in bad_patterns:
- if re.search(pattern, token):
- return token
-
- mtch = _find_match(patterns, token)
- if mtch is None:
- return token
-
- (match, _) = mtch
- keys = match.groupdict().keys()
- if 'arxiv_id' in keys:
- (front, back) = _arxiv_id_sub(match, id_to_url)
- elif 'doi' in keys:
- (front, back) = _doi_sub(match, doi_to_url)
- elif 'url' in keys:
- (front, back) = _url_sub(match, url_to_url)
- else:
- # unclear how to substitute in for this match
- return token
-
- if back:
- t_back = _transform_token(patterns, bad_patterns,
- id_to_url, doi_to_url, url_to_url, back)
- return front + Markup(t_back)
- else:
- return front
-
-
-def _arxiv_id_sub(match: Match, id_to_url: Callable[[str], str]) \
- -> Tuple[Markup, str]:
- """Return match.string transformed for a arxiv id match."""
- aid = match.group('arxiv_id')
- prefix = 'arXiv:' if match.group('arxiv_prefix') else ''
-
- if aid[-1] in _bad_endings:
- arxiv_url = id_to_url(aid)[:-1]
- anchor = aid[:-1]
- back = aid[-1] + match.string[match.end():]
- else:
- arxiv_url = id_to_url(aid)
- anchor = prefix + aid
- back = match.string[match.end():]
-
- front = match.string[0:match.start()]
- return (Markup(f'{front}{anchor}'), back)
-
-
-def _doi_sub(match: Match, doi_to_url: Callable[[str], str]) \
- ->Tuple[Markup, str]:
- """Return match.string transformed for a DOI match."""
- doi = match.group('doi')
- if(doi[-1] in _bad_endings):
- back = match.string[match.end():] + doi[-1]
- doi = doi[:-1]
- else:
- back = match.string[match.end():]
-
- quoted_doi = quote(doi, safe='/')
- doi_url = f'https://dx.doi.org/{quoted_doi}'
- doi_url = doi_to_url(doi_url)
-
- anchor = escape(doi)
- front = match.string[0:match.start()]
- return (Markup(f'{front}{anchor}'), back)
-
-
-def _url_sub(match: Match, url_to_url: Callable[[str], str]) \
- ->Tuple[Markup, str]:
- """Return match.string transformed for a URL match."""
- url = match.group('url')
- if url.startswith('https'):
- anchor = 'this https URL'
- elif url.startswith('http'):
- anchor = 'this http URL'
- elif url.startswith('ftp'):
- anchor = 'this ftp URL'
- else:
- anchor = 'this URL'
-
- front = match.string[0:match.start()]
- if url[-1] in _bad_endings:
- back = url[-1] + match.string[match.end():]
- url = url[:-1]
- else:
- back = match.string[match.end():]
-
- url = url_to_url(url)
- return (Markup(f'{front}{anchor}'), back)
-
-
-_word_split_re = re.compile(r'(\s+)')
-"""Regex to split to tokens during _to_tags.
-
-Capturing group causes the splitting spaces to be included
-in the returned list.
-"""
-
-
-def _to_tags(patterns: List[Matchable],
- bad_patterns: List[Pattern],
- id_to_url: Callable[[str], str],
- doi_to_url: Callable[[str], str],
- url_to_url: Callable[[str], str],
- text: str)-> str:
- """Split text to tokens, do _transform_token for each, return results."""
- def transform_token(tkn: str)-> str:
- return _transform_token(patterns, bad_patterns,
- id_to_url, doi_to_url, url_to_url,
- tkn)
-
- if not hasattr(text, '__html__'):
- text = Markup(escape(text))
-
- words = _word_split_re.split(text)
- for i, token in enumerate(words):
- token_2 = transform_token(token)
- if token_2 != token:
- words[i] = token_2
- result = u''.join(words)
- return Markup(result)
-
-
-def do_id_to_tags(id_to_url: Callable[[str], str],
- text: str)-> str:
- """Transform arxiv ids in text to tags."""
- return _to_tags(basic_arxiv_id_patterns,
- bad_arxiv_id_patterns,
- id_to_url, _identity, _identity,
- text)
-
-
-def do_dois_id_urls_to_tags(id_to_url: Callable[[str], str],
- doi_to_url: Callable[[str], str],
- text: str)-> str:
- """Transform DOIs, arxiv ids and URLs in text to tags."""
- return _to_tags(dois_ids_and_urls,
- bad_arxiv_id_patterns,
- id_to_url, doi_to_url, _identity,
- text)
-
-
-def do_dois_to_tags(doi_to_url: Callable[[str], str], text: str)->str:
- """Transform DOIs in text to tags."""
- return _to_tags(doi_patterns, [], _identity, doi_to_url, _identity, text)
-
-
-def do_dois_arxiv_ids_to_tags(id_to_url: Callable[[str], str],
- doi_to_url: Callable[[str], str],
- text: str)->str:
- """Transform DOIs and arXiv IDs to tags."""
- return _to_tags(doi_patterns + basic_arxiv_id_patterns,
- bad_arxiv_id_patterns,
- id_to_url, doi_to_url, _identity,
- text)
diff --git a/docs/Makefile b/docs/Makefile
new file mode 100644
index 000000000..69fe55ecf
--- /dev/null
+++ b/docs/Makefile
@@ -0,0 +1,19 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS =
+SPHINXBUILD = sphinx-build
+SOURCEDIR = source
+BUILDDIR = build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+ @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+ @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
\ No newline at end of file
diff --git a/docs/source/api/browse.config.rst b/docs/source/api/browse.config.rst
new file mode 100644
index 000000000..db15bb759
--- /dev/null
+++ b/docs/source/api/browse.config.rst
@@ -0,0 +1,7 @@
+browse.config module
+========================================
+
+.. automodule:: browse.config
+ :members:
+ :undoc-members:
+ :show-inheritance:
diff --git a/docs/source/api/browse.controllers.abs_page.rst b/docs/source/api/browse.controllers.abs_page.rst
new file mode 100644
index 000000000..060b7327f
--- /dev/null
+++ b/docs/source/api/browse.controllers.abs_page.rst
@@ -0,0 +1,7 @@
+browse.controllers.abs_page module
+========================================
+
+.. automodule:: browse.controllers.abs_page
+ :members:
+ :undoc-members:
+ :show-inheritance:
diff --git a/docs/source/api/browse.controllers.home_page.rst b/docs/source/api/browse.controllers.home_page.rst
new file mode 100644
index 000000000..7d050838e
--- /dev/null
+++ b/docs/source/api/browse.controllers.home_page.rst
@@ -0,0 +1,7 @@
+browse.controllers.home_page module
+========================================
+
+.. automodule:: browse.controllers.home_page
+ :members:
+ :undoc-members:
+ :show-inheritance:
diff --git a/docs/source/api/browse.controllers.list_page.rst b/docs/source/api/browse.controllers.list_page.rst
new file mode 100644
index 000000000..d337a6bdd
--- /dev/null
+++ b/docs/source/api/browse.controllers.list_page.rst
@@ -0,0 +1,7 @@
+browse.controllers.list_page module
+========================================
+
+.. automodule:: browse.controllers.list_page
+ :members:
+ :undoc-members:
+ :show-inheritance:
diff --git a/docs/source/api/browse.controllers.prevnext.rst b/docs/source/api/browse.controllers.prevnext.rst
new file mode 100644
index 000000000..3f6cefa63
--- /dev/null
+++ b/docs/source/api/browse.controllers.prevnext.rst
@@ -0,0 +1,7 @@
+browse.controllers.prevnext module
+========================================
+
+.. automodule:: browse.controllers.prevnext
+ :members:
+ :undoc-members:
+ :show-inheritance:
diff --git a/docs/source/api/browse.controllers.rst b/docs/source/api/browse.controllers.rst
new file mode 100644
index 000000000..0eff93e9b
--- /dev/null
+++ b/docs/source/api/browse.controllers.rst
@@ -0,0 +1,17 @@
+browse.controllers package
+==========================
+
+.. automodule:: browse.controllers
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
+Subpackages
+-----------
+
+.. toctree::
+
+ browse.controllers.abs_page
+ browse.controllers.home_page
+ browse.controllers.list_page
+ browse.controllers.prevnext
\ No newline at end of file
diff --git a/docs/source/api/browse.domain.category.rst b/docs/source/api/browse.domain.category.rst
new file mode 100644
index 000000000..58a599871
--- /dev/null
+++ b/docs/source/api/browse.domain.category.rst
@@ -0,0 +1,7 @@
+browse.domain.category module
+=============================
+
+.. automodule:: browse.domain.category
+ :members:
+ :undoc-members:
+ :show-inheritance:
diff --git a/docs/source/api/browse.domain.identifier.rst b/docs/source/api/browse.domain.identifier.rst
new file mode 100644
index 000000000..644c40ea1
--- /dev/null
+++ b/docs/source/api/browse.domain.identifier.rst
@@ -0,0 +1,7 @@
+browse.domain.identifier module
+===============================
+
+.. automodule:: browse.domain.identifier
+ :members:
+ :undoc-members:
+ :show-inheritance:
diff --git a/docs/source/api/browse.domain.institution.rst b/docs/source/api/browse.domain.institution.rst
new file mode 100644
index 000000000..434608d81
--- /dev/null
+++ b/docs/source/api/browse.domain.institution.rst
@@ -0,0 +1,7 @@
+browse.domain.institution module
+================================
+
+.. automodule:: browse.domain.institution
+ :members:
+ :undoc-members:
+ :show-inheritance:
diff --git a/docs/source/api/browse.domain.license.rst b/docs/source/api/browse.domain.license.rst
new file mode 100644
index 000000000..d6b7422c8
--- /dev/null
+++ b/docs/source/api/browse.domain.license.rst
@@ -0,0 +1,7 @@
+browse.domain.license module
+=============================
+
+.. automodule:: browse.domain.license
+ :members:
+ :undoc-members:
+ :show-inheritance:
diff --git a/docs/source/api/browse.domain.listing.rst b/docs/source/api/browse.domain.listing.rst
new file mode 100644
index 000000000..37a94f172
--- /dev/null
+++ b/docs/source/api/browse.domain.listing.rst
@@ -0,0 +1,7 @@
+browse.domain.listing module
+=============================
+
+.. automodule:: browse.domain.listing
+ :members:
+ :undoc-members:
+ :show-inheritance:
diff --git a/docs/source/api/browse.domain.metadata.rst b/docs/source/api/browse.domain.metadata.rst
new file mode 100644
index 000000000..f8cf62a04
--- /dev/null
+++ b/docs/source/api/browse.domain.metadata.rst
@@ -0,0 +1,7 @@
+browse.domain.metadata module
+===============================
+
+.. automodule:: browse.domain.metadata
+ :members:
+ :undoc-members:
+ :show-inheritance:
diff --git a/docs/source/api/browse.domain.rst b/docs/source/api/browse.domain.rst
new file mode 100644
index 000000000..1add291d4
--- /dev/null
+++ b/docs/source/api/browse.domain.rst
@@ -0,0 +1,19 @@
+browse.domain package
+=====================
+
+.. automodule:: browse.domain
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
+Submodules
+----------
+
+.. toctree::
+
+ browse.domain.category
+ browse.domain.identifier
+ browse.domain.institution
+ browse.domain.license
+ browse.domain.listing
+ browse.domain.metadata
diff --git a/docs/source/api/browse.exceptions.rst b/docs/source/api/browse.exceptions.rst
new file mode 100644
index 000000000..b228e3e2d
--- /dev/null
+++ b/docs/source/api/browse.exceptions.rst
@@ -0,0 +1,7 @@
+browse.exceptions module
+========================================
+
+.. automodule:: browse.exceptions
+ :members:
+ :undoc-members:
+ :show-inheritance:
diff --git a/docs/source/api/browse.factory.rst b/docs/source/api/browse.factory.rst
new file mode 100644
index 000000000..3de36913e
--- /dev/null
+++ b/docs/source/api/browse.factory.rst
@@ -0,0 +1,7 @@
+browse.factory module
+========================================
+
+.. automodule:: browse.factory
+ :members:
+ :undoc-members:
+ :show-inheritance:
diff --git a/docs/source/api/browse.filters.rst b/docs/source/api/browse.filters.rst
new file mode 100644
index 000000000..a45502856
--- /dev/null
+++ b/docs/source/api/browse.filters.rst
@@ -0,0 +1,7 @@
+browse.filters module
+========================================
+
+.. automodule:: browse.filters
+ :members:
+ :undoc-members:
+ :show-inheritance:
diff --git a/docs/source/api/browse.routes.rst b/docs/source/api/browse.routes.rst
new file mode 100644
index 000000000..c68fe08c3
--- /dev/null
+++ b/docs/source/api/browse.routes.rst
@@ -0,0 +1,14 @@
+browse.routes package
+=====================
+
+.. automodule:: browse.routes
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
+Submodules
+----------
+
+.. toctree::
+
+ browse.routes.ui
diff --git a/docs/source/api/browse.routes.ui.rst b/docs/source/api/browse.routes.ui.rst
new file mode 100644
index 000000000..2f1c7c58f
--- /dev/null
+++ b/docs/source/api/browse.routes.ui.rst
@@ -0,0 +1,7 @@
+browse.routes.ui module
+=======================
+
+.. automodule:: browse.routes.ui
+ :members:
+ :undoc-members:
+ :show-inheritance:
diff --git a/docs/source/api/browse.rst b/docs/source/api/browse.rst
new file mode 100644
index 000000000..076cff262
--- /dev/null
+++ b/docs/source/api/browse.rst
@@ -0,0 +1,28 @@
+browse package
+==============
+
+.. automodule:: browse
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
+Subpackages
+-----------
+
+.. toctree::
+
+ browse.controllers
+ browse.domain
+ browse.routes
+ browse.services
+ browse.util
+
+Submodules
+----------
+
+.. toctree::
+
+ browse.config
+ browse.exceptions
+ browse.factory
+ browse.filters
diff --git a/docs/source/api/browse.services.database.models.rst b/docs/source/api/browse.services.database.models.rst
new file mode 100644
index 000000000..fc7530bc1
--- /dev/null
+++ b/docs/source/api/browse.services.database.models.rst
@@ -0,0 +1,7 @@
+browse.services.database.models module
+======================================
+
+.. automodule:: browse.services.database.models
+ :members:
+ :undoc-members:
+ :show-inheritance:
diff --git a/docs/source/api/browse.services.database.rst b/docs/source/api/browse.services.database.rst
new file mode 100644
index 000000000..d2bef259f
--- /dev/null
+++ b/docs/source/api/browse.services.database.rst
@@ -0,0 +1,14 @@
+browse.services.database package
+================================
+
+.. automodule:: browse.services.database
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
+Subpackages
+-----------
+
+.. toctree::
+
+ browse.services.database.models
diff --git a/docs/source/api/browse.services.document.author_affil.rst b/docs/source/api/browse.services.document.author_affil.rst
new file mode 100644
index 000000000..1322b6ccc
--- /dev/null
+++ b/docs/source/api/browse.services.document.author_affil.rst
@@ -0,0 +1,7 @@
+browse.services.document.author_affil module
+============================================
+
+.. automodule:: browse.services.document.author_affil
+ :members:
+ :undoc-members:
+ :show-inheritance:
diff --git a/docs/source/api/browse.services.document.cache.rst b/docs/source/api/browse.services.document.cache.rst
new file mode 100644
index 000000000..684e37e14
--- /dev/null
+++ b/docs/source/api/browse.services.document.cache.rst
@@ -0,0 +1,7 @@
+browse.services.document.cache module
+=====================================
+
+.. automodule:: browse.services.document.cache
+ :members:
+ :undoc-members:
+ :show-inheritance:
diff --git a/docs/source/api/browse.services.document.metadata.rst b/docs/source/api/browse.services.document.metadata.rst
new file mode 100644
index 000000000..40a8aae95
--- /dev/null
+++ b/docs/source/api/browse.services.document.metadata.rst
@@ -0,0 +1,7 @@
+browse.services.document.metadata module
+========================================
+
+.. automodule:: browse.services.document.metadata
+ :members:
+ :undoc-members:
+ :show-inheritance:
diff --git a/docs/source/api/browse.services.document.rst b/docs/source/api/browse.services.document.rst
new file mode 100644
index 000000000..1d76663da
--- /dev/null
+++ b/docs/source/api/browse.services.document.rst
@@ -0,0 +1,17 @@
+browse.services.document package
+================================
+
+.. automodule:: browse.services.document
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
+Subpackages
+-----------
+
+.. toctree::
+
+ browse.services.document.author_affil
+ browse.services.document.cache
+ browse.services.document.metadata
+
diff --git a/docs/source/api/browse.services.listing.fake_listings.rst b/docs/source/api/browse.services.listing.fake_listings.rst
new file mode 100644
index 000000000..16ddd84e6
--- /dev/null
+++ b/docs/source/api/browse.services.listing.fake_listings.rst
@@ -0,0 +1,7 @@
+browse.services.listing.fake_listings module
+==============================================
+
+.. automodule:: browse.services.listing.fake_listings
+ :members:
+ :undoc-members:
+ :show-inheritance:
diff --git a/docs/source/api/browse.services.listing.rst b/docs/source/api/browse.services.listing.rst
new file mode 100644
index 000000000..52011e20c
--- /dev/null
+++ b/docs/source/api/browse.services.listing.rst
@@ -0,0 +1,14 @@
+browse.services.listing package
+================================
+
+.. automodule:: browse.services.listing
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
+Subpackages
+-----------
+
+.. toctree::
+
+ browse.services.listing.fake_listings
diff --git a/docs/source/api/browse.services.rst b/docs/source/api/browse.services.rst
new file mode 100644
index 000000000..d7189959b
--- /dev/null
+++ b/docs/source/api/browse.services.rst
@@ -0,0 +1,18 @@
+browse.services package
+=======================
+
+.. automodule:: browse.services
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
+Subpackages
+-----------
+
+.. toctree::
+
+ browse.services.database
+ browse.services.document
+ browse.services.listing
+ browse.services.search
+ browse.services.util
diff --git a/docs/source/api/browse.services.search.rst b/docs/source/api/browse.services.search.rst
new file mode 100644
index 000000000..a13da02c9
--- /dev/null
+++ b/docs/source/api/browse.services.search.rst
@@ -0,0 +1,14 @@
+browse.services.search package
+================================
+
+.. automodule:: browse.services.search
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
+Subpackages
+-----------
+
+.. toctree::
+
+ browse.services.search.search_authors
diff --git a/docs/source/api/browse.services.search.search_authors.rst b/docs/source/api/browse.services.search.search_authors.rst
new file mode 100644
index 000000000..d929b7ec0
--- /dev/null
+++ b/docs/source/api/browse.services.search.search_authors.rst
@@ -0,0 +1,7 @@
+browse.services.search.search_authors module
+============================================
+
+.. automodule:: browse.services.search.search_authors
+ :members:
+ :undoc-members:
+ :show-inheritance:
diff --git a/docs/source/api/browse.services.util.email.rst b/docs/source/api/browse.services.util.email.rst
new file mode 100644
index 000000000..657e9b271
--- /dev/null
+++ b/docs/source/api/browse.services.util.email.rst
@@ -0,0 +1,7 @@
+browse.services.util.email module
+============================================
+
+.. automodule:: browse.services.util.email
+ :members:
+ :undoc-members:
+ :show-inheritance:
diff --git a/docs/source/api/browse.services.util.external_refs_cits.rst b/docs/source/api/browse.services.util.external_refs_cits.rst
new file mode 100644
index 000000000..ade155369
--- /dev/null
+++ b/docs/source/api/browse.services.util.external_refs_cits.rst
@@ -0,0 +1,7 @@
+browse.services.util.external_refs_cits module
+================================================
+
+.. automodule:: browse.services.util.external_refs_cits
+ :members:
+ :undoc-members:
+ :show-inheritance:
diff --git a/docs/source/api/browse.services.util.formats.rst b/docs/source/api/browse.services.util.formats.rst
new file mode 100644
index 000000000..fa2293015
--- /dev/null
+++ b/docs/source/api/browse.services.util.formats.rst
@@ -0,0 +1,7 @@
+browse.services.util.formats module
+================================================
+
+.. automodule:: browse.services.util.formats
+ :members:
+ :undoc-members:
+ :show-inheritance:
diff --git a/docs/source/api/browse.services.util.metatags.rst b/docs/source/api/browse.services.util.metatags.rst
new file mode 100644
index 000000000..6623c2d9b
--- /dev/null
+++ b/docs/source/api/browse.services.util.metatags.rst
@@ -0,0 +1,7 @@
+browse.services.util.metatags module
+================================================
+
+.. automodule:: browse.services.util.metatags
+ :members:
+ :undoc-members:
+ :show-inheritance:
diff --git a/docs/source/api/browse.services.util.response_headers.rst b/docs/source/api/browse.services.util.response_headers.rst
new file mode 100644
index 000000000..15daf9438
--- /dev/null
+++ b/docs/source/api/browse.services.util.response_headers.rst
@@ -0,0 +1,7 @@
+browse.services.util.response_headers module
+================================================
+
+.. automodule:: browse.services.util.response_headers
+ :members:
+ :undoc-members:
+ :show-inheritance:
diff --git a/docs/source/api/browse.services.util.rst b/docs/source/api/browse.services.util.rst
new file mode 100644
index 000000000..a20abf25c
--- /dev/null
+++ b/docs/source/api/browse.services.util.rst
@@ -0,0 +1,19 @@
+browse.services.util package
+================================
+
+.. automodule:: browse.services.util
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
+Subpackages
+-----------
+
+.. toctree::
+
+ browse.services.util.email
+ browse.services.util.external_refs_cits
+ browse.services.util.formats
+ browse.services.util.metatags
+ browse.services.util.response_headers
+ browse.services.util.tex2utf
diff --git a/docs/source/api/browse.services.util.tex2utf.rst b/docs/source/api/browse.services.util.tex2utf.rst
new file mode 100644
index 000000000..070b736dd
--- /dev/null
+++ b/docs/source/api/browse.services.util.tex2utf.rst
@@ -0,0 +1,7 @@
+browse.services.util.tex2utf module
+================================================
+
+.. automodule:: browse.services.util.tex2utf
+ :members:
+ :undoc-members:
+ :show-inheritance:
diff --git a/docs/source/api/browse.util.rst b/docs/source/api/browse.util.rst
new file mode 100644
index 000000000..2491304cb
--- /dev/null
+++ b/docs/source/api/browse.util.rst
@@ -0,0 +1,7 @@
+browse.util module
+========================================
+
+.. automodule:: browse.util
+ :members:
+ :undoc-members:
+ :show-inheritance:
diff --git a/docs/source/api/modules.rst b/docs/source/api/modules.rst
new file mode 100644
index 000000000..797a8bd73
--- /dev/null
+++ b/docs/source/api/modules.rst
@@ -0,0 +1,7 @@
+browse
+======
+
+.. toctree::
+ :maxdepth: 4
+
+ browse
\ No newline at end of file
diff --git a/docs/architecture.rst b/docs/source/architecture.rst
similarity index 99%
rename from docs/architecture.rst
rename to docs/source/architecture.rst
index a361d771d..e1445bd71 100644
--- a/docs/architecture.rst
+++ b/docs/source/architecture.rst
@@ -1,5 +1,5 @@
-Browse Application
-******************
+Browse Application Architecture
+*******************************
The browse application provides reader-facing views onto arXiv documents and
their metadata. "Browse" functionality encompasses the abstract page, content
diff --git a/docs/source/conf.py b/docs/source/conf.py
new file mode 100644
index 000000000..e941fe2ed
--- /dev/null
+++ b/docs/source/conf.py
@@ -0,0 +1,190 @@
+# -*- coding: utf-8 -*-
+#
+# Configuration file for the Sphinx documentation builder.
+#
+# This file does only contain a selection of the most common options. For a
+# full list see the documentation:
+# http://www.sphinx-doc.org/en/master/config
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+# import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+import os
+import sys
+sys.path.insert(0, os.path.abspath('.'))
+sys.path.append(os.path.abspath('..'))
+sys.path.append(os.path.abspath('../..'))
+
+# -- Project information -----------------------------------------------------
+
+project = 'arXiv Browse Service'
+copyright = '2019, arXiv-NG Team'
+author = 'arXiv-NG Team'
+
+# The short X.Y version
+version = ''
+# The full version, including alpha/beta/rc tags
+release = '0.2.1'
+
+
+# -- General configuration ---------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+ 'sphinx.ext.autodoc',
+ 'sphinx.ext.intersphinx',
+ 'sphinx.ext.coverage',
+ 'sphinx.ext.viewcode',
+ 'sphinx.ext.githubpages',
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+# source_suffix = ['.rst', '.md']
+source_suffix = '.rst'
+
+# The master toctree document.
+master_doc = 'index'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = None
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = []
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = None
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages. See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'alabaster'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further. For a list of options available for each theme, see the
+# documentation.
+#
+# html_theme_options = {}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+# Custom sidebar templates, must be a dictionary that maps document names
+# to template names.
+#
+# The default sidebars (for documents that don't match any pattern) are
+# defined by theme itself. Builtin themes are using these templates by
+# default: ``['localtoc.html', 'relations.html', 'sourcelink.html',
+# 'searchbox.html']``.
+#
+# html_sidebars = {}
+
+
+# -- Options for HTMLHelp output ---------------------------------------------
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'arXivBrowseServicedoc'
+
+
+# -- Options for LaTeX output ------------------------------------------------
+
+latex_elements = {
+ # The paper size ('letterpaper' or 'a4paper').
+ #
+ # 'papersize': 'letterpaper',
+
+ # The font size ('10pt', '11pt' or '12pt').
+ #
+ # 'pointsize': '10pt',
+
+ # Additional stuff for the LaTeX preamble.
+ #
+ # 'preamble': '',
+
+ # Latex figure (float) alignment
+ #
+ # 'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+# author, documentclass [howto, manual, or own class]).
+latex_documents = [
+ (master_doc, 'arXivBrowseService.tex', 'arXiv Browse Service Documentation',
+ 'arXiv-NG Team', 'manual'),
+]
+
+
+# -- Options for manual page output ------------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+ (master_doc, 'arxivbrowseservice', 'arXiv Browse Service Documentation',
+ [author], 1)
+]
+
+
+# -- Options for Texinfo output ----------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+# dir menu entry, description, category)
+texinfo_documents = [
+ (master_doc, 'arXivBrowseService', 'arXiv Browse Service Documentation',
+ author, 'arXivBrowseService', 'One line description of project.',
+ 'Miscellaneous'),
+]
+
+
+# -- Options for Epub output -------------------------------------------------
+
+# Bibliographic Dublin Core info.
+epub_title = project
+
+# The unique identifier of the text. This can be a ISBN number
+# or the project homepage.
+#
+# epub_identifier = ''
+
+# A unique identification for the text.
+#
+# epub_uid = ''
+
+# A list of files that should not be packed into the epub file.
+epub_exclude_files = ['search.html']
+
+
+# -- Extension configuration -------------------------------------------------
+
+# -- Options for intersphinx extension ---------------------------------------
+
+# Example configuration for intersphinx: refer to the Python standard library.
+intersphinx_mapping = {'https://docs.python.org/': None}
diff --git a/docs/source/index.rst b/docs/source/index.rst
new file mode 100644
index 000000000..9f01e357a
--- /dev/null
+++ b/docs/source/index.rst
@@ -0,0 +1,23 @@
+.. arXiv Browse Service documentation master file, created by
+ sphinx-quickstart on Wed Feb 13 16:27:27 2019.
+ You can adapt this file completely to your liking, but it should at least
+ contain the root `toctree` directive.
+
+Welcome to arXiv Browse Service's documentation!
+================================================
+
+.. toctree::
+ :maxdepth: 3
+ :caption: Contents:
+
+ architecture
+ api/modules
+
+
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`
diff --git a/tests/data/abs_files/ftp/arxiv/papers/1902/1902.11195.abs b/tests/data/abs_files/ftp/arxiv/papers/1902/1902.11195.abs
new file mode 100644
index 000000000..8e2c26b7f
--- /dev/null
+++ b/tests/data/abs_files/ftp/arxiv/papers/1902/1902.11195.abs
@@ -0,0 +1,46 @@
+------------------------------------------------------------------------------
+\\
+arXiv:1902.11195
+From: Israel Mardor
+Date: Thu, 28 Feb 2019 16:26:35 GMT (2136kb,D)
+
+Title: A Novel Method for the Measurement of Half-Lives and Decay Branching
+ Ratios of Exotic Nuclei
+Authors: Ivan Miskun (1), Timo Dickel (1 and 2), Israel Mardor (3 and 4),
+ Christine Hornung (1), Daler Amanbayev (1), Samuel Ayet San Andr\'es (1 and
+ 2), Julian Bergmann (1), Jens Ebert (1), Hans Geissel (1 and 2), Magdalena
+ G\'orska (2), Florian Greiner (1), Emma Haettner (2), Wolfgang R. Pla{\ss} (1
+ and 2), Sivaji Purushothaman (2), Christoph Scheidenberger (1 and 2),
+ Ann-Kathrin Rink (1), Helmut Weick (2), Soumya Bagchi (1 and 2 and 6), Paul
+ Constantin (5), Satbir Kaur (6), Wayne Lippert (1), Bo Mei (5), Iain Moore
+ (7), Jan-Hendrick Otto (1), Stephane Pietri (2), Ilkka Pohjalainen (7),
+ Andrej Prochazka (2), Christoph Rappold (1 and 2), Moritz P. Reiter (1 and
+ 8), Yoshiki K. Tanaka (2), John S. Winfield (2), for the Super-FRS Experiment
+ Collaboration ((1) Justus-Liebig-Universit\"at, Gie{\ss}en, Germany (2) GSI
+ Helmholtzzentrum f\"ur Schwerionenforschung GmbH, Darmstadt, Germany (3) Tel
+ Aviv University, Tel Aviv, Israel (4) Soreq Nuclear Research Center, Yavne,
+ Israel (5) IFIN-HH/ELI-NP, M\u{a}gurele - Bucharest, Romania (6) Saint Mary's
+ University, Halifax, Canada (7) University of Jyv\"askyl\"a, Jyv\"askyl\"a,
+ Finland (8) TRIUMF, Vancouver, Canada)
+Categories: nucl-ex
+Comments: 11 pages, 9 figures, 3 tables. Submitted to European Physics Journal
+ A
+License: http://arxiv.org/licenses/nonexclusive-distrib/1.0/
+\\
+ A novel method for simultaneous measurement of masses, Q-values, isomer
+excitation energies, half-lives and decay branching ratios of exotic nuclei has
+been demonstrated. The method includes first use of a stopping cell as an ion
+trap, combining containment of precursors and decay-recoils for variable
+durations in a cryogenic stopping cell (CSC), and afterwards the identification
+and counting of them by a multiple-reflection time-of-flight mass spectrometer
+(MR-TOF-MS). Feasibility has been established by recording the decay and growth
+of $^{216}$Po and $^{212}$Pb (alpha decay) and of $^{119m2}$Sb (t$_{1/2}$ =
+850$\pm$90 ms) and $^{119g}$Sb (isomer transition), obtaining half-lives and
+branching ratios consistent with literature values. Hardly any
+non-nuclear-decay losses have been observed in the CSC for up to $\sim$10
+seconds, which exhibits its extraordinary cleanliness. For $^{119}$Sb, this is
+the first direct measurement of the ground and second isomeric state masses,
+resolving the discrepancies in previous excitation energy data. These results
+pave the way for the measurement of branching ratios of exotic nuclei with
+multiple decay channels.
+\\
diff --git a/tests/data/abs_files/orig/math-ph/papers/0702031v1.abs b/tests/data/abs_files/orig/math-ph/papers/0702031v1.abs
deleted file mode 100644
index 5809d6dec..000000000
--- a/tests/data/abs_files/orig/math-ph/papers/0702031v1.abs
+++ /dev/null
@@ -1,17 +0,0 @@
-------------------------------------------------------------------------------
-\\
-arXiv:math-ph/0702031
-From: Paulo Mendon\c{c}a
-Date: Fri, 9 Feb 2007 20:39:05 GMT (8kb)
-
-Title: Probability Distribution of Curvatures of Isosurfaces in Gaussian Random
- Fields
-Authors: Paulo R. S. Mendonca, Rahul Bhotika and James V. Miller
-Categories: math-ph.MP
-Comments: Seven pages, six references
-MSC-class: 60D05
-\\
- An expression for the joint probability distribution of the principal
-curvatures at an arbitrary point in the ensemble of isosurfaces defined on
-isotropic Gaussian random fields on Rn is derived.
-\\
diff --git a/tests/data/browse.db b/tests/data/browse.db
index 3ca1a9d37..10847cc3e 100644
Binary files a/tests/data/browse.db and b/tests/data/browse.db differ
diff --git a/tests/data/daily_stats b/tests/data/daily_stats
new file mode 100644
index 000000000..31ea233b8
--- /dev/null
+++ b/tests/data/daily_stats
@@ -0,0 +1 @@
+total_papers 1234567
diff --git a/tests/data/db/sql/arXiv_in_category.sql b/tests/data/db/sql/arXiv_in_category.sql
new file mode 100644
index 000000000..8e691811a
--- /dev/null
+++ b/tests/data/db/sql/arXiv_in_category.sql
@@ -0,0 +1,9 @@
+INSERT INTO `arXiv_in_category` VALUES (1,'cond-mat','stat-mech',0),(1,'math','CO',1),(1,'math','MP',0),(1,'math-ph','',0);
+INSERT INTO `arXiv_in_category` VALUES (2,'math','AG',1),(2,'math','NT',0);
+INSERT INTO `arXiv_in_category` VALUES (3,'math','DG',1);
+INSERT INTO `arXiv_in_category` VALUES (4,'math','CT',0),(4,'math','KT',1);
+INSERT INTO `arXiv_in_category` VALUES (5,'gr-qc','',1);
+INSERT INTO `arXiv_in_category` VALUES (6,'gr-qc','',1);
+INSERT INTO `arXiv_in_category` VALUES (7,'cond-mat','mtrl-sci',0),(7,'cond-mat','supr-con',1);
+INSERT INTO `arXiv_in_category` VALUES (422353,'cs','IT',1),(422353,'math','IT',0);
+INSERT INTO `arXiv_in_category` VALUES (504934,'hep-ex','',1);
diff --git a/tests/data/db/sql/arXiv_stats_hourly.sql b/tests/data/db/sql/arXiv_stats_hourly.sql
new file mode 100644
index 000000000..9f75799cf
--- /dev/null
+++ b/tests/data/db/sql/arXiv_stats_hourly.sql
@@ -0,0 +1,5 @@
+-- full day's data for 2019-01-012
+INSERT INTO `arXiv_stats_hourly` VALUES ('2019-01-02',0,3,'N',53845),('2019-01-02',0,2,'N',48406),('2019-01-02',0,4,'N',48828),('2019-01-02',0,1,'N',51429),('2019-01-02',0,1,'A',3),('2019-01-02',1,3,'N',56725),('2019-01-02',1,2,'N',53353),('2019-01-02',1,2,'A',3),('2019-01-02',1,4,'N',48812),('2019-01-02',1,1,'N',51958),('2019-01-02',1,1,'A',5),('2019-01-02',2,3,'N',59992),('2019-01-02',2,2,'N',61270),('2019-01-02',2,1,'N',54667),('2019-01-02',2,1,'A',2),('2019-01-02',2,4,'N',50583),('2019-01-02',3,3,'N',66846),('2019-01-02',3,2,'N',63584),('2019-01-02',3,4,'N',62662),('2019-01-02',3,4,'A',2),('2019-01-02',3,1,'N',61056),('2019-01-02',3,1,'A',4),('2019-01-02',4,3,'N',59514),('2019-01-02',4,1,'N',53321),('2019-01-02',4,1,'A',2),('2019-01-02',4,2,'N',63700),('2019-01-02',4,4,'N',55601),('2019-01-02',5,3,'N',62566),('2019-01-02',5,3,'A',2),('2019-01-02',5,1,'N',57549),('2019-01-02',5,1,'A',12),('2019-01-02',5,4,'N',56604),('2019-01-02',5,2,'N',61001),('2019-01-02',5,2,'A',7),('2019-01-02',6,1,'N',56351),('2019-01-02',6,1,'A',5),('2019-01-02',6,3,'N',63059),('2019-01-02',6,4,'N',54324),('2019-01-02',6,2,'N',59236),('2019-01-02',6,2,'A',3),('2019-01-02',7,3,'N',59573),('2019-01-02',7,2,'N',61090),('2019-01-02',7,4,'N',54211),('2019-01-02',7,1,'N',57702),('2019-01-02',7,1,'A',2),('2019-01-02',8,3,'N',70691),('2019-01-02',8,2,'N',72144),('2019-01-02',8,4,'N',73142),('2019-01-02',8,1,'N',32346),('2019-01-02',8,1,'A',2),('2019-01-02',9,3,'N',65412),('2019-01-02',9,4,'N',60758),('2019-01-02',9,1,'N',56908),('2019-01-02',9,1,'A',4),('2019-01-02',9,2,'N',65283),('2019-01-02',9,2,'A',2),('2019-01-02',10,3,'N',68411),('2019-01-02',10,4,'N',60375),('2019-01-02',10,1,'N',59069),('2019-01-02',10,1,'A',2),('2019-01-02',10,2,'N',63476),('2019-01-02',11,3,'N',63408),('2019-01-02',11,3,'A',2),('2019-01-02',11,1,'N',54618),('2019-01-02',11,1,'A',19),('2019-01-02',11,4,'N',58673),('2019-01-02',11,2,'N',58308),('2019-01-02',12,3,'N',58050),('2019-01-02',12,1,'N',51337),('2019-01-02',12,1,'A',8),('2019-01-02',12,2,'N',57397),('2019-01-02',12,4,'N',54196),('2019-01-02',13,2,'N',51752),('2019-01-02',13,1,'N',45888),('2019-01-02',13,1,'A',2),('2019-01-02',13,3,'N',52890),('2019-01-02',13,4,'N',52570),('2019-01-02',14,4,'N',47099),('2019-01-02',14,1,'N',42504),('2019-01-02',14,1,'A',2),('2019-01-02',14,3,'N',49769),('2019-01-02',14,2,'N',47754),('2019-01-02',15,3,'N',51613),('2019-01-02',15,1,'N',42363),('2019-01-02',15,1,'A',6),('2019-01-02',15,2,'N',46362),('2019-01-02',15,4,'N',44812),('2019-01-02',16,3,'N',44893),('2019-01-02',16,4,'N',42667),('2019-01-02',16,2,'N',46088),('2019-01-02',16,1,'N',39196),('2019-01-02',16,1,'A',2),('2019-01-02',17,3,'N',42756),('2019-01-02',17,2,'N',44652),('2019-01-02',17,2,'A',4),('2019-01-02',17,1,'N',37109),('2019-01-02',17,1,'A',12),('2019-01-02',17,4,'N',40573),('2019-01-02',17,4,'A',5),('2019-01-02',18,3,'N',39766),('2019-01-02',18,4,'N',35739),('2019-01-02',18,4,'A',3),('2019-01-02',18,1,'N',34420),('2019-01-02',18,1,'A',5),('2019-01-02',18,2,'N',40016),('2019-01-02',19,4,'N',38157),('2019-01-02',19,3,'N',39220),('2019-01-02',19,2,'N',42100),('2019-01-02',19,1,'N',34453),('2019-01-02',19,1,'A',2),('2019-01-02',20,4,'N',53578),('2019-01-02',20,3,'N',58880),('2019-01-02',20,2,'N',61952),('2019-01-02',20,1,'N',56445),('2019-01-02',20,1,'A',2),('2019-01-02',21,4,'N',59692),('2019-01-02',21,2,'N',62295),('2019-01-02',21,2,'A',2843),('2019-01-02',21,3,'N',69497),('2019-01-02',21,1,'N',63502),('2019-01-02',21,1,'A',3116),('2019-01-02',22,4,'N',48579),('2019-01-02',22,2,'N',53989),('2019-01-02',22,3,'N',55692),('2019-01-02',22,1,'N',54425),('2019-01-02',22,1,'A',2570),('2019-01-02',23,4,'N',46401),('2019-01-02',23,4,'A',327),('2019-01-02',23,2,'N',48631),('2019-01-02',23,2,'A',1521),('2019-01-02',23,3,'N',50670),('2019-01-02',23,1,'N',51057),('2019-01-02',23,1,'A',1852);
+
+-- partial day's data for 2019-03-19
+INSERT INTO `arXiv_stats_hourly` VALUES ('2019-03-19',0,4,'N',52840),('2019-03-19',0,4,'A',3),('2019-03-19',0,2,'N',53704),('2019-03-19',0,3,'N',55367),('2019-03-19',0,3,'A',276),('2019-03-19',0,1,'N',63019),('2019-03-19',0,1,'A',6035),('2019-03-19',1,3,'N',57338),('2019-03-19',1,3,'A',667),('2019-03-19',1,2,'N',56893),('2019-03-19',1,4,'N',58308),('2019-03-19',1,4,'A',209),('2019-03-19',1,1,'N',69393),('2019-03-19',1,1,'A',7040),('2019-03-19',2,3,'N',65608),('2019-03-19',2,3,'A',282),('2019-03-19',2,2,'N',64133),('2019-03-19',2,4,'N',65662),('2019-03-19',2,4,'A',617),('2019-03-19',2,1,'N',78694),('2019-03-19',2,1,'A',3468),('2019-03-19',3,4,'N',41070),('2019-03-19',3,4,'A',163),('2019-03-19',3,2,'N',75674),('2019-03-19',3,2,'A',68),('2019-03-19',3,3,'N',74601),('2019-03-19',3,3,'A',540),('2019-03-19',3,1,'N',82732),('2019-03-19',3,1,'A',1012),('2019-03-19',4,4,'N',82631),('2019-03-19',4,4,'A',641),('2019-03-19',4,3,'N',82636),('2019-03-19',4,3,'A',709),('2019-03-19',4,2,'N',74236),('2019-03-19',4,1,'N',81817),('2019-03-19',4,1,'A',1339),('2019-03-19',5,1,'N',83531),('2019-03-19',5,1,'A',502),('2019-03-19',5,4,'N',80857),('2019-03-19',5,4,'A',303),('2019-03-19',5,3,'N',80824),('2019-03-19',5,3,'A',198),('2019-03-19',5,2,'N',78596),('2019-03-19',6,1,'N',75874),('2019-03-19',6,1,'A',977),('2019-03-19',6,3,'N',76941),('2019-03-19',6,2,'N',74382),('2019-03-19',6,4,'N',75905),('2019-03-19',7,2,'N',75759),('2019-03-19',7,1,'N',82056),('2019-03-19',7,1,'A',1908),('2019-03-19',7,3,'N',77887),('2019-03-19',7,4,'N',79405),('2019-03-19',8,1,'N',89503),('2019-03-19',8,1,'A',1975),('2019-03-19',8,2,'N',79577),('2019-03-19',8,4,'N',88666),('2019-03-19',8,3,'N',84634);
diff --git a/tests/data/db/sql/arXiv_stats_monthly_downloads.sql b/tests/data/db/sql/arXiv_stats_monthly_downloads.sql
new file mode 100644
index 000000000..eb9d7f18f
--- /dev/null
+++ b/tests/data/db/sql/arXiv_stats_monthly_downloads.sql
@@ -0,0 +1 @@
+INSERT INTO `arXiv_stats_monthly_downloads` VALUES ('1994-01-01',0),('1994-02-01',1863),('1994-03-01',7048),('1994-04-01',6814),('1994-05-01',9601),('1994-06-01',11007),('1994-07-01',9488),('1994-08-01',10789),('1994-09-01',12872),('1994-10-01',16260),('1994-11-01',22392),('1994-12-01',20070),('1995-01-01',24801),('1995-02-01',28607),('1995-03-01',35418),('1995-04-01',34109),('1995-05-01',35232),('1995-06-01',52415),('1995-07-01',68341),('1995-08-01',95848),('1995-09-01',114550),('1995-10-01',146524),('1995-11-01',151624),('1995-12-01',132908),('1996-01-01',160742),('1996-02-01',186528),('1996-03-01',193169),('1996-04-01',198175),('1996-05-01',221716),('1996-06-01',213072),('1996-07-01',230395),('1996-08-01',222063),('1996-09-01',251249),('1996-10-01',287532),('1996-11-01',263905),('1996-12-01',225928),('1997-01-01',250816),('1997-02-01',230357),('1997-03-01',239669),('1997-04-01',258302),('1997-05-01',267683),('1997-06-01',261706),('1997-07-01',270550),('1997-08-01',239698),('1997-09-01',287158),('1997-10-01',326150),('1997-11-01',296375),('1997-12-01',288487),('1998-01-01',321044),('1998-02-01',337514),('1998-03-01',407658),('1998-04-01',384583),('1998-05-01',409663),('1998-06-01',404232),('1998-07-01',412267),('1998-08-01',382223),('1998-09-01',444628),('1998-10-01',510790),('1998-11-01',537888),('1998-12-01',474617),('1999-01-01',508224),('1999-02-01',464452),('1999-03-01',511665),('1999-04-01',510014),('1999-05-01',536821),('1999-06-01',547141),('1999-07-01',544712),('1999-08-01',491688),('1999-09-01',568449),('1999-10-01',582934),('1999-11-01',665411),('1999-12-01',627284),('2000-01-01',640945),('2000-02-01',725616),('2000-03-01',751154),('2000-04-01',697884),('2000-05-01',785650),('2000-06-01',745325),('2000-07-01',728494),('2000-08-01',721467),('2000-09-01',724899),('2000-10-01',828870),('2000-11-01',925260),('2000-12-01',844998),('2001-01-01',935314),('2001-02-01',972075),('2001-03-01',1055737),('2001-04-01',1013386),('2001-05-01',1059624),('2001-06-01',902582),('2001-07-01',846039),('2001-08-01',865395),('2001-09-01',834963),('2001-10-01',969584),('2001-11-01',966553),('2001-12-01',1145430),('2002-01-01',1034782),('2002-02-01',982129),('2002-03-01',1175829),('2002-04-01',1321376),('2002-05-01',1423935),('2002-06-01',1236836),('2002-07-01',1348311),('2002-08-01',1270201),('2002-09-01',1516847),('2002-10-01',1675586),('2002-11-01',1515041),('2002-12-01',1343960),('2003-01-01',1588531),('2003-02-01',1572793),('2003-03-01',1711279),('2003-04-01',1620843),('2003-05-01',1703389),('2003-06-01',1601569),('2003-07-01',1586144),('2003-08-01',1504793),('2003-09-01',1820701),('2003-10-01',2117036),('2003-11-01',2033657),('2003-12-01',2119846),('2004-01-01',2109079),('2004-02-01',2293511),('2004-03-01',2221885),('2004-04-01',2100595),('2004-05-01',2301904),('2004-06-01',2036838),('2004-07-01',1986919),('2004-08-01',1997330),('2004-09-01',2191867),('2004-10-01',1927789),('2004-11-01',1978766),('2004-12-01',1767095),('2005-01-01',1804435),('2005-02-01',1803019),('2005-03-01',2068684),('2005-04-01',2251800),('2005-05-01',2136947),('2005-06-01',1951444),('2005-07-01',1852432),('2005-08-01',1852131),('2005-09-01',2120662),('2005-10-01',2346256),('2005-11-01',2406316),('2005-12-01',2360494),('2006-01-01',1701784),('2006-02-01',1737794),('2006-03-01',2298683),('2006-04-01',1997694),('2006-05-01',2214161),('2006-06-01',1801721),('2006-07-01',1401479),('2006-08-01',1629104),('2006-09-01',1768658),('2006-10-01',1982461),('2006-11-01',1930368),('2006-12-01',1663348),('2007-01-01',2029217),('2007-02-01',2066818),('2007-03-01',2134854),('2007-04-01',1979950),('2007-05-01',2224762),('2007-06-01',2224194),('2007-07-01',2051276),('2007-08-01',1822372),('2007-09-01',1998723),('2007-10-01',2446094),('2007-11-01',2531762),('2007-12-01',2159429),('2008-01-01',2665839),('2008-02-01',2557077),('2008-03-01',2848757),('2008-04-01',2810180),('2008-05-01',2958761),('2008-06-01',2706793),('2008-07-01',2730226),('2008-08-01',2450019),('2008-09-01',3278451),('2008-10-01',2937969),('2008-11-01',3066092),('2008-12-01',2638793),('2009-01-01',2598159),('2009-02-01',2644797),('2009-03-01',3104692),('2009-04-01',2837273),('2009-05-01',2877553),('2009-06-01',2792334),('2009-07-01',2667955),('2009-08-01',2538405),('2009-09-01',2907441),('2009-10-01',3167801),('2009-11-01',2912668),('2009-12-01',2723394),('2010-01-01',3116867),('2010-02-01',2992581),('2010-03-01',3459004),('2010-04-01',3292459),('2010-05-01',3293995),('2010-06-01',3335937),('2010-07-01',3098569),('2010-08-01',2879133),('2010-09-01',3144001),('2010-10-01',3684560),('2010-11-01',3829250),('2010-12-01',3775736),('2011-01-01',3439915),('2011-02-01',3339054),('2011-03-01',4117851),('2011-04-01',3659749),('2011-05-01',3997359),('2011-06-01',3707592),('2011-07-01',3771042),('2011-08-01',3774913),('2011-09-01',4973988),('2011-10-01',5563412),('2011-11-01',4962036),('2011-12-01',4433059),('2012-01-01',4796243),('2012-02-01',5078432),('2012-03-01',5516536),('2012-04-01',5423005),('2012-05-01',5067807),('2012-06-01',5073183),('2012-07-01',5624113),('2012-08-01',6040066),('2012-09-01',5372689),('2012-10-01',5904272),('2012-11-01',5373061),('2012-12-01',4549990),('2013-01-01',5665096),('2013-02-01',5136321),('2013-03-01',6121433),('2013-04-01',5552584),('2013-05-01',6184081),('2013-06-01',5033496),('2013-07-01',5251361),('2013-08-01',4985314),('2013-09-01',5319428),('2013-10-01',6146267),('2013-11-01',5953050),('2013-12-01',5464673),('2014-01-01',6701768),('2014-02-01',6175371),('2014-03-01',7371741),('2014-04-01',7207160),('2014-05-01',6814576),('2014-06-01',6650843),('2014-07-01',6934640),('2014-08-01',6368537),('2014-09-01',7129287),('2014-10-01',9150599),('2014-11-01',10771479),('2014-12-01',9877539),('2015-01-01',10211128),('2015-02-01',10375127),('2015-03-01',12826855),('2015-04-01',11289672),('2015-05-01',11182540),('2015-06-01',12568030),('2015-07-01',9585300),('2015-08-01',9180739),('2015-09-01',10741431),('2015-10-01',13882451),('2015-11-01',14200441),('2015-12-01',10286698),('2016-01-01',16227193),('2016-02-01',14547141),('2016-03-01',16144490),('2016-04-01',14330220),('2016-05-01',14042457),('2016-06-01',14344946),('2016-07-01',12885660),('2016-08-01',11795801),('2016-09-01',13958435),('2016-10-01',11202303),('2016-11-01',11492492),('2016-12-01',12136801),('2017-01-01',12150875),('2017-02-01',12870204),('2017-03-01',14319435),('2017-04-01',15628477),('2017-05-01',14894937),('2017-06-01',14372378),('2017-07-01',14808475),('2017-08-01',15915733),('2017-09-01',16999890),('2017-10-01',19433648),('2017-11-01',18885644),('2017-12-01',17361253),('2018-01-01',18445580),('2018-02-01',17704444),('2018-03-01',20149872),('2018-04-01',19709932),('2018-05-01',19710718),('2018-06-01',18088369),('2018-07-01',18132320),('2018-08-01',17906302),('2018-09-01',17418053),('2018-10-01',19443085),('2018-11-01',23040835),('2018-12-01',18694165),('2019-01-01',19281479);
diff --git a/tests/data/db/sql/arXiv_stats_monthly_submissions.sql b/tests/data/db/sql/arXiv_stats_monthly_submissions.sql
new file mode 100644
index 000000000..23932c0d6
--- /dev/null
+++ b/tests/data/db/sql/arXiv_stats_monthly_submissions.sql
@@ -0,0 +1 @@
+INSERT INTO `arXiv_stats_monthly_submissions` VALUES ('1991-07-01',2,-2),('1991-08-01',28,-1),('1991-09-01',58,0),('1991-10-01',76,0),('1991-11-01',64,0),('1991-12-01',78,0),('1992-01-01',193,-105),('1992-02-01',134,-10),('1992-03-01',120,-3),('1992-04-01',225,-41),('1992-05-01',237,-11),('1992-06-01',237,-5),('1992-07-01',296,-22),('1992-08-01',227,-4),('1992-09-01',336,-21),('1992-10-01',401,-29),('1992-11-01',453,-16),('1992-12-01',404,-10),('1993-01-01',400,-36),('1993-02-01',428,-16),('1993-03-01',504,-9),('1993-04-01',505,-17),('1993-05-01',538,-7),('1993-06-01',536,-16),('1993-07-01',643,-35),('1993-08-01',541,-25),('1993-09-01',518,-16),('1993-10-01',678,-36),('1993-11-01',714,-16),('1993-12-01',738,-15),('1994-01-01',610,-28),('1994-02-01',653,-19),('1994-03-01',753,-24),('1994-04-01',738,-39),('1994-05-01',841,-18),('1994-06-01',888,-32),('1994-07-01',863,-23),('1994-08-01',766,-20),('1994-09-01',884,-33),('1994-10-01',932,-23),('1994-11-01',1114,-39),('1994-12-01',1055,-33),('1995-01-01',933,-36),('1995-02-01',977,-36),('1995-03-01',1168,-37),('1995-04-01',913,-31),('1995-05-01',1110,-22),('1995-06-01',1195,-27),('1995-07-01',1017,-25),('1995-08-01',1030,-22),('1995-09-01',1133,-29),('1995-10-01',1229,-23),('1995-11-01',1162,-28),('1995-12-01',1147,-37),('1996-01-01',1051,-22),('1996-02-01',1076,-23),('1996-03-01',1169,-23),('1996-04-01',1220,-49),('1996-05-01',1358,-42),('1996-06-01',1311,-34),('1996-07-01',1424,-32),('1996-08-01',1394,-19),('1996-09-01',1474,-21),('1996-10-01',1520,-30),('1996-11-01',1402,-27),('1996-12-01',1467,-33),('1997-01-01',1308,-31),('1997-02-01',1362,-34),('1997-03-01',1411,-24),('1997-04-01',1468,-34),('1997-05-01',1581,-32),('1997-06-01',1709,-31),('1997-07-01',1804,-41),('1997-08-01',1426,-15),('1997-09-01',1882,-37),('1997-10-01',2007,-26),('1997-11-01',1717,-32),('1997-12-01',1949,-31),('1998-01-01',1723,-11),('1998-02-01',1668,0),('1998-03-01',1911,-3),('1998-04-01',1747,-12),('1998-05-01',1918,-2),('1998-06-01',2089,-6),('1998-07-01',2101,-12),('1998-08-01',1826,-6),('1998-09-01',2414,-9),('1998-10-01',2330,-33),('1998-11-01',2242,-14),('1998-12-01',2203,-7),('1999-01-01',1850,-23),('1999-02-01',1919,-3),('1999-03-01',2394,-15),('1999-04-01',2156,-9),('1999-05-01',2221,-17),('1999-06-01',2427,-5),('1999-07-01',2414,-23),('1999-08-01',2139,-1),('1999-09-01',2502,-12),('1999-10-01',2571,-6),('1999-11-01',2518,-29),('1999-12-01',2593,-6),('2000-01-01',2365,-6),('2000-02-01',2366,0),('2000-03-01',2600,-9),('2000-04-01',2077,-9),('2000-05-01',2727,-9),('2000-06-01',2439,-3),('2000-07-01',2461,-16),('2000-08-01',2613,-1),('2000-09-01',2522,-7),('2000-10-01',2913,-3),('2000-11-01',2864,-11),('2000-12-01',2654,-1),('2001-01-01',2591,-14),('2001-02-01',2429,-3),('2001-03-01',2713,-8),('2001-04-01',2606,-6),('2001-05-01',2912,-4),('2001-06-01',2871,-5),('2001-07-01',2760,-3),('2001-08-01',2422,0),('2001-09-01',2548,-3),('2001-10-01',3404,-4),('2001-11-01',3240,-1),('2001-12-01',2718,-19),('2002-01-01',2716,-4),('2002-02-01',2567,-2),('2002-03-01',2666,-3),('2002-04-01',2853,-4),('2002-05-01',3089,-2),('2002-06-01',2695,0),('2002-07-01',3269,-6),('2002-08-01',2725,0),('2002-09-01',3314,-2),('2002-10-01',3531,0),('2002-11-01',3440,-3),('2002-12-01',3256,-35),('2003-01-01',2926,0),('2003-02-01',2881,0),('2003-03-01',3018,0),('2003-04-01',3102,0),('2003-05-01',3267,0),('2003-06-01',3463,0),('2003-07-01',3418,0),('2003-08-01',2741,0),('2003-09-01',3747,-12),('2003-10-01',3841,0),('2003-11-01',3349,0),('2003-12-01',3661,0),('2004-01-01',3071,0),('2004-02-01',3277,-1),('2004-03-01',3603,0),('2004-04-01',3369,-1),('2004-05-01',3553,0),('2004-06-01',3710,0),('2004-07-01',3679,0),('2004-08-01',3294,0),('2004-09-01',3939,0),('2004-10-01',4086,0),('2004-11-01',4152,0),('2004-12-01',3994,0),('2005-01-01',3493,0),('2005-02-01',3250,0),('2005-03-01',3881,0),('2005-04-01',3693,0),('2005-05-01',3779,0),('2005-06-01',3984,0),('2005-07-01',3839,0),('2005-08-01',3775,0),('2005-09-01',4345,-16),('2005-10-01',4439,0),('2005-11-01',4294,0),('2005-12-01',4083,0),('2006-01-01',3858,0),('2006-02-01',3520,0),('2006-03-01',4213,0),('2006-04-01',3474,0),('2006-05-01',4204,0),('2006-06-01',4139,0),('2006-07-01',4197,0),('2006-08-01',4064,0),('2006-09-01',4275,0),('2006-10-01',5133,0),('2006-11-01',4854,0),('2006-12-01',4296,0),('2007-01-01',4653,0),('2007-02-01',4164,0),('2007-03-01',4493,0),('2007-04-01',4003,0),('2007-05-01',4684,0),('2007-06-01',4484,0),('2007-07-01',4681,0),('2007-08-01',4414,0),('2007-09-01',4682,0),('2007-10-01',5945,0),('2007-11-01',5029,0),('2007-12-01',4406,0),('2008-01-01',4970,0),('2008-02-01',4463,0),('2008-03-01',4519,0),('2008-04-01',4898,0),('2008-05-01',4836,0),('2008-06-01',4980,0),('2008-07-01',5139,0),('2008-08-01',4160,0),('2008-09-01',5287,0),('2008-10-01',5773,0),('2008-11-01',4774,0),('2008-12-01',5116,0),('2009-01-01',4975,0),('2009-02-01',4903,0),('2009-03-01',5547,0),('2009-04-01',4930,0),('2009-05-01',4955,0),('2009-06-01',5614,0),('2009-07-01',5606,0),('2009-08-01',4597,0),('2009-09-01',5696,0),('2009-10-01',5957,0),('2009-11-01',5730,0),('2009-12-01',5537,0),('2010-01-01',5471,0),('2010-02-01',5048,0),('2010-03-01',6133,0),('2010-04-01',5602,0),('2010-05-01',5737,0),('2010-06-01',5963,0),('2010-07-01',5521,0),('2010-08-01',5399,0),('2010-09-01',6232,0),('2010-10-01',6304,0),('2010-11-01',6676,0),('2010-12-01',6045,0),('2011-01-01',6081,0),('2011-02-01',5774,0),('2011-03-01',6286,0),('2011-04-01',5711,0),('2011-05-01',6374,0),('2011-06-01',6357,0),('2011-07-01',6046,0),('2011-08-01',6331,0),('2011-09-01',6937,0),('2011-10-01',6930,0),('2011-11-01',7319,0),('2011-12-01',6432,0),('2012-01-01',6687,0),('2012-02-01',6685,0),('2012-03-01',6903,0),('2012-04-01',6739,0),('2012-05-01',7092,0),('2012-06-01',7121,0),('2012-07-01',7358,0),('2012-08-01',6592,0),('2012-09-01',6630,0),('2012-10-01',8452,0),('2012-11-01',7370,0),('2012-12-01',6974,0),('2013-01-01',7750,0),('2013-02-01',7317,0),('2013-03-01',7476,0),('2013-04-01',8135,0),('2013-05-01',7516,0),('2013-06-01',6944,0),('2013-07-01',8447,0),('2013-08-01',6833,0),('2013-09-01',7995,0),('2013-10-01',8658,0),('2013-11-01',7692,0),('2013-12-01',7878,0),('2014-01-01',8294,0),('2014-02-01',7374,0),('2014-03-01',8154,0),('2014-04-01',7855,0),('2014-05-01',7975,0),('2014-06-01',7874,0),('2014-07-01',8549,0),('2014-08-01',7119,0),('2014-09-01',8676,0),('2014-10-01',8871,0),('2014-11-01',8006,0),('2014-12-01',8770,0),('2015-01-01',7912,0),('2015-02-01',8054,0),('2015-03-01',9191,0),('2015-04-01',8367,0),('2015-05-01',8172,0),('2015-06-01',9217,0),('2015-07-01',8995,0),('2015-08-01',7983,0),('2015-09-01',9318,0),('2015-10-01',9223,0),('2015-11-01',9472,0),('2015-12-01',9376,0),('2016-01-01',8251,0),('2016-02-01',9142,0),('2016-03-01',9746,0),('2016-04-01',8948,0),('2016-05-01',9792,0),('2016-06-01',9644,0),('2016-07-01',8911,0),('2016-08-01',9016,0),('2016-09-01',9869,0),('2016-10-01',10100,0),('2016-11-01',10362,0),('2016-12-01',9599,0),('2017-01-01',9186,0),('2017-02-01',8910,0),('2017-03-01',11008,0),('2017-04-01',9029,0),('2017-05-01',11194,0),('2017-06-01',10297,0),('2017-07-01',9980,0),('2017-08-01',9854,0),('2017-09-01',10517,0),('2017-10-01',11627,0),('2017-11-01',11589,0),('2017-12-01',10332,0),('2018-01-01',10609,0),('2018-02-01',10593,0),('2018-03-01',11560,0),('2018-04-01',11352,0),('2018-05-01',12595,0),('2018-06-01',11568,0),('2018-07-01',11938,0),('2018-08-01',10870,0),('2018-09-01',11171,0),('2018-10-01',13446,0),('2018-11-01',12941,0),('2018-12-01',11973,0),('2019-01-01',11537,0);
diff --git a/tests/data/db/sql/arXiv_trackback_pings.sql b/tests/data/db/sql/arXiv_trackback_pings.sql
index ba0b01357..8fd0575f8 100644
--- a/tests/data/db/sql/arXiv_trackback_pings.sql
+++ b/tests/data/db/sql/arXiv_trackback_pings.sql
@@ -1,3 +1,4 @@
+-- 0808.4142
INSERT INTO `arXiv_trackback_pings` VALUES (695736,504934,'Omega b: the new baryon nailed by D0 « A Quantum Diaries Survivor',' pulling this new result off! You can read the details of the analysis in this paper. ','http://dorigo.wordpress.com/2008/09/19/omega-b-the-new-baryon-nailed-by-d0/','','arxiv1.library.cornell.edu','128.84.158.114',1221882921,0,0,1221883202,'accepted',61);
INSERT INTO `arXiv_trackback_pings` VALUES (751383,504934,'Nit-Picking On The Omega_b Discovery','quark and two strange quarks- has been recently published by the DZERO collaboration. Their paper claims to observe the so-far-unseen','http://www.scientificblogging.com/quantum_diaries_survivor/nitpicking_omega_b_discovery','A Quantum Diaries Survivor','arxiv1.library.cornell.edu','128.84.158.114',1243482527,0,0,1243484559,'accepted',NULL);
INSERT INTO `arXiv_trackback_pings` VALUES (751384,504934,'Nit-Picking On The Omega_b Baryon: Part II','the pains to check the calculation hinted at on the scientific paper where the discovery is claimed, which was sent to the ','http://www.scientificblogging.com/quantum_diaries_survivor/nitpicking_omega_b_baryon_part_ii','A Quantum Diaries Survivor','arxiv1.library.cornell.edu','128.84.158.114',1243482588,0,0,1243484559,'accepted',NULL);
@@ -12,5 +13,4 @@ INSERT INTO `arXiv_trackback_pings` VALUES (1,504934,'More On The Omega B Baryon
-- This next one is an intentional duplicate, but with different url, and not accepted
INSERT INTO `arXiv_trackback_pings` VALUES (2,504934,'Foo baz','\"Observation of the','http://en.wikipedia.org/wiki/Zevatron','Wikipedia','arxiv1.library.cornell.edu','128.84.158.114',1228752458,0,0,0,'rejected',NULL);
-INSERT INTO `arXiv_trackback_pings` VALUES (99999999,99999998,'Nothing here','Really nothing here ','http://www.example.org','On Nothing','arxiv1.library.cornell.edu','128.84.158.114',1258488297,0,0,1248558336,'accepted',NULL);
-
+INSERT INTO `arXiv_trackback_pings` VALUES (99999999,504934,'Nothing here','Really nothing here ','http://www.example.org','On Nothing','arxiv1.library.cornell.edu','128.84.158.114',1258488294,0,0,1248558336,'accepted',NULL);
diff --git a/tests/legacy_comparison/archive_config.py b/tests/legacy_comparison/archive_config.py
new file mode 100644
index 000000000..964ccf083
--- /dev/null
+++ b/tests/legacy_comparison/archive_config.py
@@ -0,0 +1,12 @@
+from response_comparisons import compare_status
+
+res_comparisons = [ compare_status ]
+text_comparisons = []
+html_comparisons = []
+
+def ng_id_to_url_fn(id:str)->str:
+ return 'http://localhost:5000/archive/' + id
+
+def legacy_id_to_url_fn(id:str)->str:
+ return 'https://arxiv.org/archive/' + id
+
diff --git a/tests/legacy_comparison/archives.txt b/tests/legacy_comparison/archives.txt
new file mode 100644
index 000000000..df3f31942
--- /dev/null
+++ b/tests/legacy_comparison/archives.txt
@@ -0,0 +1,20 @@
+astro-ph
+cond-mat
+gr-qc
+hep-ex
+hep-lat
+hep-ph
+hep-th
+math-ph
+nlin
+nucl-ex
+nucl-th
+physics
+quant-ph
+math
+cs
+q-bio
+q-fin
+stat
+eess
+econ
diff --git a/tests/legacy_comparison/html_comparisons.py b/tests/legacy_comparison/html_comparisons.py
index 9d7a5eda7..f130db375 100644
--- a/tests/legacy_comparison/html_comparisons.py
+++ b/tests/legacy_comparison/html_comparisons.py
@@ -1,10 +1,10 @@
-from functools import partial
+from functools import partial, update_wrapper
from typing import Callable, List
import re
-from tests.legacy_comparison.abstract_comparisons import lev_similarity
-from tests.legacy_comparison.comparison_types import html_arg_dict, BadResult
+from abstract_comparisons import lev_similarity
+from comparison_types import html_arg_dict, BadResult
from bs4 import BeautifulSoup, element
@@ -15,7 +15,9 @@ def html_similarity(html_arg: html_arg_dict) -> BadResult:
html_arg['legacy_html'].prettify()
)
if sim < 0.69:
- return BadResult(html_arg['paper_id'], f"html_pretty_sim for {html_arg['paper_id']} = {sim}")
+ return BadResult(html_arg['id'],
+ "html_similarity",
+ f"html_pretty_sim for {html_arg['id']} = {sim}")
return None
@@ -34,7 +36,7 @@ def to_label(tr): return tr.find('td', 'label').contents
if ng_labels == legacy_labels:
return None
else:
- return BadResult(html_arg['paper_id'],
+ return BadResult(html_arg['id'],
"Metadata field included on NG do not match those from legacy" +
f"NG: {ng_labels} Legacy: {legacy_labels}")
@@ -69,14 +71,14 @@ def _element_similarity(name: str,
if required:
if len(ng) == 0 and len(legacy) == 0:
- return BadResult(html_arg['paper_id'], name,
- f"Missing field {name} for {html_arg['paper_id']} from NG and Legacy")
+ return BadResult(html_arg['id'], name,
+ f"Missing field {name} for {html_arg['id']} from NG and Legacy")
if len(ng) == 0:
- return BadResult(html_arg['paper_id'], name,
- f"Missing field {name} for {html_arg['paper_id']} from NG")
+ return BadResult(html_arg['id'], name,
+ f"Missing field {name} for {html_arg['id']} from NG")
if len(legacy) == 0:
- return BadResult(html_arg['paper_id'], name,
- f"Missing field {name} for {html_arg['paper_id']} from legacy")
+ return BadResult(html_arg['id'], name,
+ f"Missing field {name} for {html_arg['id']} from legacy")
if check_counts and (len(legacy) != len(ng)):
if ng:
@@ -88,8 +90,8 @@ def _element_similarity(name: str,
else:
legacy_ele_txt = 'MISSING'
- return BadResult(html_arg['paper_id'], name,
- f"bad counts for {name} for {html_arg['paper_id']} ng: {len(ng)} legacy: {len(legacy)}",
+ return BadResult(html_arg['id'], name,
+ f"bad counts for {name} for {html_arg['id']} ng: {len(ng)} legacy: {len(legacy)}",
legacy_ele_txt, ng_ele_txt)
ng_ele_txt = ''
@@ -102,9 +104,11 @@ def _element_similarity(name: str,
if sim < min_sim:
msg = f"Elements did not meet min similarity of {min_sim}"
- return BadResult(html_arg['paper_id'], name, msg, legacy_ele_txt,
+ return BadResult(html_arg['id'], name, msg, legacy_ele_txt,
ng_ele_txt, sim)
- return None
+
+ msg = f"GOOD: Elements did meet min similarity of {min_sim}"
+ return BadResult(html_arg['id'], name, msg, '','', sim)
else:
if not required:
return None
@@ -116,7 +120,7 @@ def _element_similarity(name: str,
msg = 'zero elements detected: ' \
+ f'legacy length was {len(legacy)}; ng length was {len(ng)} '
- return BadResult(html_arg['paper_id'], name, msg, legacy_ele_txt,
+ return BadResult(html_arg['id'], name, msg, legacy_ele_txt,
ng_ele_txt, 0.0)
@@ -197,3 +201,32 @@ def ex_strip(eles: List[BeautifulSoup]):
bookmarks_similarity = partial(_element_similarity, 'extra bookmarks div',
lambda bs: ex_strip(bs.select('.bookmarks')),
0.9, False, True)
+
+################# /archive checks ################################
+
+archive_h1_similarity = partial(_element_similarity, 'top heading',
+ lambda bs: ex_strip(bs.select('#content > h1')),
+ 0.99, True, True)
+
+archive_browse = partial(_element_similarity, 'browse',
+ lambda bs: ex_strip(bs.select('#content > ul > li:nth-child(1)')),
+ 0.99, True, True)
+
+archive_catchup = partial(_element_similarity, 'archive catchup',
+ lambda bs: ex_strip(bs.select('#content > ul > li:nth-child(2)')),
+ 0.99, True, True)
+
+archive_search= partial(_element_similarity, 'archive_search',
+ lambda bs: ex_strip(bs.select('#content > ul > li:nth-child(3)')),
+ 0.99, True, True)
+
+archive_by_year= partial(_element_similarity, 'archive_by_year',
+ lambda bs: ex_strip(bs.select('#content > ul > li:nth-child(4)')),
+ 0.99, True, True)
+
+
+archive_bogus= partial(_element_similarity, 'bogus_should_fail',
+ lambda bs: ex_strip(bs.select('.bogusClass')),
+ 0.99, True, True)
+
+
diff --git a/tests/legacy_comparison/page_comparison.py b/tests/legacy_comparison/page_comparison.py
new file mode 100644
index 000000000..de577dad1
--- /dev/null
+++ b/tests/legacy_comparison/page_comparison.py
@@ -0,0 +1,295 @@
+import argparse
+import itertools
+import sys
+import traceback
+import os
+import re
+from functools import partial
+import multiprocessing_on_dill as mp
+from typing import Callable, Iterator, List, Set, Tuple, Dict, Any
+import gzip
+import logging
+import json
+
+import requests
+from bs4 import BeautifulSoup
+
+# BDC34: some how I need this under pipenv to get to browse, not sure why
+sys.path.append('')
+sys.setrecursionlimit(10000)
+
+from comparison_types import res_comparison_fn, \
+ text_comparison_fn, html_comparison_fn, res_arg_dict, text_arg_dict, \
+ html_arg_dict, BadResult
+from html_comparisons import html_similarity, metadata_fields_similarity, \
+ archive_h1_similarity,archive_catchup,archive_search,archive_by_year,archive_browse,\
+ archive_bogus
+
+from response_comparisons import compare_status
+from text_comparisons import text_similarity
+
+import archive_config
+
+
+""" Script to compare pages from NG and beta.arxiv.org
+
+To run this I do:
+Open terminal:
+cd arxiv-browse
+pipenv sync
+FLASK_APP=app.py FLASK_DEBUG=1 pipenv run flask run
+
+In another terminal:
+cd arxiv-browse
+pipenv sync
+pipenv shell
+python tests/legacy_comparison/page_comparison.py
+
+To reset the analysis to start over, add the `--reset` arg.
+To run a short test add '--short' arg.
+
+To skip ancillary file comparisons: '--skip-ancillary'
+
+Improvements:
+ Better reporting format, right now the comparisons produce just strings.
+"""
+
+logging.basicConfig(filename="page_comparison.log", level=logging.DEBUG)
+
+#This just renames None nicely, they are tests that passed
+SUCCESS = None
+
+LOG_FILE_NAME = 'legacy_comparison.org'
+
+
+# TODO abstract this or move this out
+VISITED_FILE_NAME = 'visited.log'
+
+def ident(x):
+ return x
+
+configs = {
+ 'archive': { # I'd like to do something with modules but it doesn't pickle.
+ 'comparisons': [compare_status,
+ html_similarity,
+ archive_h1_similarity,
+ archive_browse,
+ archive_catchup,
+ archive_search,
+ archive_by_year,
+ ],
+ 'ng_id_to_url_fn': archive_config.ng_id_to_url_fn,
+ 'legacy_id_to_url_fn': archive_config.legacy_id_to_url_fn,
+ 'ng_txt_trans_fn': ident,
+ 'legacy_txt_trans_fn': ident,
+ }
+}
+
+# id file is one id per line
+def _id_generator_from_file(path: str, excluded: List[str])->Iterator[str]:
+ if 'gzip' in path or 'gz' in path:
+ with gzip.open(path, 'rt') as f:
+ for line in f:
+ aid = line.strip()
+ if aid not in excluded:
+ logging.debug(f'yielding id {aid}')
+ yield aid
+ else:
+ with open(path, 'rt') as f:
+ for line in f:
+ aid = line.strip()
+ if aid not in excluded:
+ logging.debug(f'yielding id {aid}')
+ yield aid
+
+#TODO generalize
+# Should end with /
+#ng_abs_base_url = 'http://localhost:5000/abs/'
+#ng_abs_base_url = 'https://beta.arxiv.org/abs/'
+
+#TODO generalize
+# Should end with /
+#legacy_abs_base_url = 'https://beta.arxiv.org/abs_classic/'
+
+def fetch_pages(config: Dict, id: str) ->Dict:
+ """Fetch NG and Legacy."""
+ ng_url = config['ng_id_to_url_fn'](id)
+ legacy_url = config['legacy_id_to_url_fn'](id)
+
+ headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'}
+ ng_res = requests.get(ng_url, headers=headers)
+ legacy_res = requests.get(legacy_url, headers=headers)
+
+ ng_text = config['ng_txt_trans_fn'](ng_res.text)
+ legacy_text = config['ng_txt_trans_fn'](legacy_res.text)
+
+ res_dict: res_arg_dict = {
+ 'id': id,
+ 'ng_url': ng_url,
+ 'legacy_url': legacy_url,
+ }
+
+ # to avoid json problems
+ with_html = {
+ **res_dict,
+ 'ng_res': ng_res,
+ 'legacy_res': legacy_res,
+ 'ng_text': ng_text,
+ 'legacy_text': legacy_text,
+ 'ng_html': BeautifulSoup(ng_text, 'html.parser'),
+ 'legacy_html': BeautifulSoup(legacy_text, 'html.parser'),
+ }
+ return (res_dict, compare_response(config,
+ with_html))
+
+
+def compare_response(config: Dict,
+ res_args: res_arg_dict) -> Iterator[BadResult]:
+ """Do the response comparisions, the kick off the text comparisions."""
+# protected_comps = [protect(fn) for fn in config['comparisons']]
+ protected_comps = config['comparisons']
+ logging.debug(f"about to do compares for {res_args['id']}")
+ return filter(SUCCESS, [fn(res_args) for fn in protected_comps])
+
+
+def multi_ws_to_single_ws(txt:str)->str:
+ return ' '.join(txt.split()) #white space to single spaces
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser(description='Compare ng pages to legacy pages')
+ parser.add_argument('--idfile', default=False, )
+ parser.add_argument('--reset', default=False, const=True,
+ action='store_const', dest='reset')
+ parser.add_argument('--short', default=False, const=True,
+ action='store_const', dest='short')
+ parser.add_argument('--config', default=False)
+ args = parser.parse_args()
+
+ print('Starting config')
+ if args.config not in configs.keys():
+ raise ValueError(f"No config named '{args.config}' choose one of [{' '.join(configs.keys())}]")
+ else:
+ print(f'Using config {args.config}')
+ active_config = configs[args.config]
+ print('done with config')
+
+ visited: Set[str] = []
+ if args.reset:
+ print('Restarting analysis and deleting logs!')
+ if os.path.exists(LOG_FILE_NAME):
+ os.remove(LOG_FILE_NAME)
+ if os.path.exists(VISITED_FILE_NAME):
+ os.remove(VISITED_FILE_NAME)
+ else:
+ if os.path.exists(VISITED_FILE_NAME):
+ print('Continuing analysis')
+ with open(VISITED_FILE_NAME, 'r') as visited_fh:
+ visited = {line.rstrip() for line in visited_fh.readlines()}
+
+ ids = _id_generator_from_file(args.idfile, excluded=visited)
+
+ if args.short:
+ n=0
+ total = 10
+ logging.info(f'Doing short list of {n}')
+ def done()->bool:
+ nonlocal n
+ if n >= total:
+ return True
+ n = n + 1
+ return False
+ else:
+ def done()->bool:
+ return False
+
+ f_then_c = partial( fetch_pages, active_config)
+
+ with open(VISITED_FILE_NAME, 'a', buffering=1) as visited_fh:
+ logging.debug(f'Opened {VISITED_FILE_NAME} to find already visited ids')
+ with open(LOG_FILE_NAME, 'w', buffering=1)as report_fh:
+ logging.debug(f'Opened {LOG_FILE_NAME} to write report to')
+ with mp.Pool(4) as pool:
+ completed_jobs \
+ = pool.imap_unordered(f_then_c, ids)
+
+ def done_job(job):
+ (res_dict, bad_results) = job
+ logging.debug(f"completed {res_dict['id']}")
+ visited_fh.write(f"{res_dict['id']}\n")
+ write_comparison_org(report_fh, (res_dict, list(bad_results)))
+ if done():
+ logging.info("done and existing")
+ exit(0)
+
+ [done_job(job) for job in completed_jobs]
+
+
+
+def protect(fn: Callable[[Any],BadResult] )-> Callable[[Any],BadResult]:
+ """Return function that will not throw"""
+ def protected(res_args:Dict) -> BadResult:
+ # noinspection PyBroadException
+ try:
+ return fn(res_args)
+ except Exception as ex:
+ return BadResult(res_args['id'], "name unknown", traceback.format_exc())
+ return protected
+
+
+def _serialize_obj(obj):
+ """JSON serializer for objects not serializable by default json code"""
+ return obj.__dict__
+
+def write_comparison(report_fh, result: Tuple[Dict, List[BadResult]]) -> None:
+ (config, bad_results) = result
+ logging.debug("writing report for %s", config['id'])
+ report_fh.write( json.dumps(config, sort_keys=True) + "\n")
+ if bad_results:
+ report_fh.write( json.dumps(bad_results, sort_keys=True, default=_serialize_obj) + "\n")
+ else:
+ report_fh.write("no bad results\n")
+
+def write_comparison_org(report_fh, result: Tuple[Dict, List[BadResult]]) -> None:
+ (config, bad_results) = result
+ logging.debug("writing report for %s", config['id'])
+ report_fh.write(f"* {config['id']} \n")
+
+ report_fh.write(f"** config for {config['id']}\n")
+ report_fh.write(json.dumps(config, sort_keys=True) + "\n")
+
+ report_fh.write("** Results\n")
+ if bad_results:
+ for result in bad_results:
+ report_fh.write(f"*** {result.comparison} \n")
+ #report_fh.write( json.dumps(result, sort_keys=True, default=_serialize_obj) + "\n")
+ report_fh.write(result.message + "\n")
+ report_fh.write(f"**** NG value: \n{result.ng}\n")
+ report_fh.write(f"**** Legacy value: \n{result.legacy}\n")
+ else:
+ report_fh.write("No bad results\n")
+
+
+def format_bad_result(bad: BadResult)->str:
+ rpt = f"** {bad.comparison}\n" \
+ f"{bad.message} "
+ if bad.similarity:
+ rpt = rpt + f"sim: {bad.similarity}\n"
+ else:
+ rpt = rpt + "\n"
+
+ if bad.legacy or bad.ng:
+ rpt = rpt + f"Legacy: '{bad.legacy}'\nNG: '{bad.ng}'\n"
+
+ return rpt
+
+
+#this is just to get around passing the config to the pool
+def dict_from_module(module):
+ context = {}
+ for setting in required_keys:
+ context[setting] = getattr(module, setting)
+ return context
+
+if __name__ == '__main__':
+ main()
diff --git a/tests/legacy_comparison/response_comparisons.py b/tests/legacy_comparison/response_comparisons.py
index 83143e2a8..e6d48e552 100644
--- a/tests/legacy_comparison/response_comparisons.py
+++ b/tests/legacy_comparison/response_comparisons.py
@@ -1,4 +1,4 @@
-from tests.legacy_comparison.comparison_types import res_arg_dict, BadResult
+from comparison_types import res_arg_dict, BadResult
def compare_status(res_arg: res_arg_dict) -> BadResult:
diff --git a/tests/legacy_comparison/text_comparisons.py b/tests/legacy_comparison/text_comparisons.py
index 50740cf2f..4dd019e18 100644
--- a/tests/legacy_comparison/text_comparisons.py
+++ b/tests/legacy_comparison/text_comparisons.py
@@ -1,5 +1,5 @@
-from tests.legacy_comparison.abstract_comparisons import lev_similarity
-from tests.legacy_comparison.comparison_types import text_arg_dict, BadResult
+from abstract_comparisons import lev_similarity
+from comparison_types import text_arg_dict, BadResult
def text_similarity(text_arg: text_arg_dict) -> BadResult:
diff --git a/tests/test_404.py b/tests/test_404.py
index 1cb9cb4a8..5babaeae5 100644
--- a/tests/test_404.py
+++ b/tests/test_404.py
@@ -1,10 +1,11 @@
import unittest
-import logging
+import logging
from app import app
-
-class It_Should_be_404_Test(unittest.TestCase):
-
+
+
+class Test_404(unittest.TestCase):
+
def setUp(self):
"""Disable logging to avoid messy output during testing"""
import logging
@@ -15,11 +16,10 @@ def setUp(self):
app.config['APPLICATION_ROOT'] = ''
self.app = app.test_client()
- def its_should_be_404_test(self):
+ def test_it_should_be_404(self):
rv = self.app.get('/abs?archive=foo&papernum=1234567')
self.assertEqual(rv.status_code, 404)
rv = self.app.get('/abs?0704.0600')
self.assertEqual(rv.status_code, 404,
'singleton case for new IDs not supported')
-
diff --git a/tests/test_archive.py b/tests/test_archive.py
new file mode 100644
index 000000000..b4b80b7e8
--- /dev/null
+++ b/tests/test_archive.py
@@ -0,0 +1,71 @@
+import unittest
+
+from app import app
+
+
+class BrowseTest(unittest.TestCase):
+ def setUp(self):
+ app.testing = True
+ app.config["APPLICATION_ROOT"] = ""
+ self.app = app.test_client()
+
+ def test_astroph_archive(self):
+ rv = self.app.get("/archive/astro-ph")
+ self.assertEqual(rv.status_code, 200)
+ self.assertIn('Expires', rv.headers, 'Should have expires header')
+
+ src = rv.data.decode("utf-8")
+ self.assertIn("Astrophysics", src)
+ self.assertIn("/year/astro-ph/92", src)
+ self.assertIn("/year/astro-ph/19", src)
+
+ self.assertIn(
+ "Astrophysics of Galaxies",
+ src,
+ "Subcategories of astro-ph should be on archive page",
+ )
+ self.assertIn(
+ "Earth and Planetary Astrophysics",
+ src,
+ "Subcategories of astro-ph should be on archive page",
+ )
+
+ def test_list(self):
+ rv = self.app.get("/archive/list")
+ self.assertEqual(rv.status_code, 200)
+ src = rv.data.decode("utf-8")
+
+ self.assertIn("Astrophysics", src)
+ self.assertIn("astro-ph", src)
+
+ self.assertIn("Materials Theory", src)
+ self.assertIn("mtrl-th", src)
+
+ rv = self.app.get("/archive/bogus-archive")
+ self.assertEqual(rv.status_code, 404)
+
+ def test_subsumed_archive(self):
+ rv = self.app.get("/archive/comp-lg")
+ self.assertEqual(rv.status_code, 404)
+ src = rv.data.decode("utf-8")
+
+ self.assertIn("Computer Science", src)
+ self.assertIn("cs.CL", src)
+
+ rv = self.app.get("/archive/acc-phys")
+ self.assertEqual(rv.status_code, 200)
+ src = rv.data.decode("utf-8")
+
+ self.assertIn("Accelerator Physics", src)
+ self.assertIn("physics.acc-ph", src)
+
+ def test_single_archive(self):
+ rv = self.app.get("/archive/hep-ph")
+ self.assertEqual(rv.status_code, 200)
+ src = rv.data.decode("utf-8")
+
+ self.assertIn("High Energy Physics", src)
+ self.assertNotIn("Categories within", src)
+
+
+
diff --git a/tests/test_author_affil.py b/tests/test_author_affil.py
deleted file mode 100644
index e240ad166..000000000
--- a/tests/test_author_affil.py
+++ /dev/null
@@ -1,291 +0,0 @@
-"""Tests for author and affiliation parsing."""
-from unittest import TestCase
-
-from browse.services.document.author_affil import parse_author_affil, split_authors
-from browse.services.document.metadata import AbsMetaSession
-from tests import path_of_for_test
-
-
-class TestAuthorAffiliationParsing(TestCase):
-
- def test_split_authors(self):
- self.assertListEqual(split_authors('Simeon Warner'),
- ['Simeon Warner'])
-
- self.assertListEqual(split_authors('The DELPHI Collaboration, J. Abdallah, et al'),
- ['The DELPHI Collaboration', ',', 'J. Abdallah', ',', 'et al'])
-
- self.assertListEqual(split_authors('BELLE Collaboration: A Person, Nother Person'),
- ['BELLE Collaboration', ':', 'A Person', ',', 'Nother Person'])
-
- self.assertListEqual(split_authors('Simeon Warner, Herbert Van de Sompel'),
- ['Simeon Warner', ',', 'Herbert Van de Sompel'])
-
- self.assertListEqual(
- split_authors('An Author'),
- ['An Author'],
- 'single author'
- )
-
- self.assertListEqual(
- split_authors(''),
- [],
- 'empty author'
- )
-
- self.assertListEqual(
- split_authors('An Author (affil)'),
- ['An Author', '(affil)'],
- 'single author with affil'
- )
- self.assertListEqual(
- split_authors('An Author (affil)'),
- ['An Author', '(affil)'],
- 'single author with affil'
- )
- self.assertListEqual(
- split_authors('An Author and Another P. H. J. Author (affil)'),
- ['An Author', ',', 'Another P. H. J. Author', '(affil)'],
- 'double author with affil'
- )
- self.assertListEqual(
- split_authors(
- 'John Von Neumann, Herbert Van de Sompel, Fred Bloggs, Jr, et al'),
- ['John Von Neumann', ',', 'Herbert Van de Sompel',
- ',', 'Fred Bloggs, Jr', ',', 'et al'],
- 'multiple with prefixes and suffixes'
- )
- self.assertListEqual(
- split_authors('sum won ( whatever affil data unmunged )'),
- ['sum won', '( whatever affil data unmunged )'],
- 'one author, two labs'
- )
- self.assertListEqual(
- split_authors('sum won(1,2)((1)lab a,(2)lab b)'),
- ['sum won', '(1,2)', '((1)lab a,(2)lab b)'],
- 'one author, two labs'
- )
-
- def test_parse_author_affil_basic(self):
- self.assertListEqual(parse_author_affil(
- 'Simeon Warner'), [['Warner', 'Simeon', '']])
-
- self.assertListEqual(
- parse_author_affil('Simeon Warner Jr'),
- [['Warner', 'Simeon', 'Jr']])
-
- self.assertListEqual(
- parse_author_affil('Simeon Warner Jr.'),
- [['Warner', 'Simeon', 'Jr.']])
-
- self.assertListEqual(
- parse_author_affil('Simeon Warner Sr'),
- [['Warner', 'Simeon', 'Sr']])
-
- self.assertListEqual(
- parse_author_affil('Simeon Warner Sr.'),
- [['Warner', 'Simeon', 'Sr.']])
-
- self.assertListEqual(
- parse_author_affil('SM Warner'),
- [['Warner', 'SM', '']])
-
- self.assertListEqual(
- parse_author_affil('SM. Warner'),
- [['Warner', 'SM.', '']])
-
- def test_parse_author_affil_basic2(self):
- self.assertListEqual(
- parse_author_affil('S.M. Warner'),
- [['Warner', 'S. M.', '']])
-
- self.assertListEqual(
- parse_author_affil('John Von Neumann'),
- [['Von Neumann', 'John', '']])
-
- self.assertListEqual(
- parse_author_affil('Herbert Van de Sompel'),
- [['Van de Sompel', 'Herbert', '']])
-
- self.assertListEqual(
- parse_author_affil('del Norte'),
- [['Norte', 'del', '']])
-
- self.assertListEqual(
- parse_author_affil('Fred del Norte'),
- [['del Norte', 'Fred', '']])
-
- self.assertListEqual(
- parse_author_affil('BELLE'),
- [['BELLE', '', '']])
-
- self.assertListEqual(
- parse_author_affil('BELLE Collaboration: A Person, Nother Person'),
- [
- ['BELLE Collaboration', '', ''],
- ['Person', 'A', ''],
- ['Person', 'Nother', '']
- ])
-
- self.assertListEqual(parse_author_affil('The DELPHI Collaboration, J. Abdallah, et al'),
- [['The DELPHI Collaboration', '', ''], ['Abdallah', 'J.', '']])
-
- self.assertListEqual(parse_author_affil('Ali Vaziri Astaneh, Federico Fuentes'),
- [['Vaziri Astaneh', 'Ali', ''],['Fuentes', 'Federico', '']])
-
- def test_parse_author_affil_with_affiliations(self):
- self.assertListEqual(
- parse_author_affil('sum won (lab a)'),
- [['won', 'sum', '', 'lab a']])
-
- self.assertListEqual(
- parse_author_affil('sum won (lab a; lab b)'),
- [['won', 'sum', '', 'lab a; lab b']])
-
- self.assertListEqual(
- parse_author_affil('sum won (lab a, lab b)'),
- [['won', 'sum', '', 'lab a, lab b']])
-
- self.assertListEqual(
- parse_author_affil('sum won (1,2) ( (1) lab a, (2) lab b)'),
- [['won', 'sum', '', 'lab a', 'lab b']])
-
- self.assertListEqual(
- parse_author_affil('sum won(1,2)((1)lab a,(2)lab b)'),
- [['won', 'sum', '', 'lab a', 'lab b']])
-
- self.assertListEqual(
- parse_author_affil('a.b.first, c.d.second (affil)'),
- [['first', 'a. b.', '', 'affil'], ['second', 'c. d.', '', 'affil']])
-
- self.assertListEqual(
- parse_author_affil('a.b.first (affil), c.d.second (affil)'),
- [['first', 'a. b.', '', 'affil'], ['second', 'c. d.', '', 'affil']])
-
- self.assertListEqual(
- parse_author_affil(
- 'a.b.first, c.d.second (1), e.f.third, g.h.forth (2,3) ((1) affil1, (2) affil2, (3) affil3)'
- ),
- [
- ['first', 'a. b.', '', 'affil1'],
- ['second', 'c. d.', '', 'affil1'],
- ['third', 'e. f.', '', 'affil2', 'affil3'],
- ['forth', 'g. h.', '', 'affil2', 'affil3']
- ])
-
- self.assertListEqual(
- parse_author_affil((
- "QUaD collaboration: S. Gupta (1), P. Ade (1), J. Bock (2,3), M. Bowden "
- "(1,4), M. L. Brown (5), G. Cahill (6), P. G. Castro (7,8), S. Church (4), T. "
- "Culverhouse (9), R. B. Friedman (9), K. Ganga (10), W. K. Gear (1), J. "
- "Hinderks (5,11), J. Kovac (3), A. E. Lange (4), E. Leitch (2,3), S. J. "
- "Melhuish (12), Y. Memari (7), J. A. Murphy (6), A. Orlando (1,3), C. "
- "O'Sullivan (6), L. Piccirillo (12), C. Pryke (9), N. Rajguru (1,13), B. "
- "Rusholme (4,14), R. Schwarz (9), A. N. Taylor (7), K. L. Thompson (4), A. H. "
- "Turner (1), E. Y. S. Wu (4), M. Zemcov (1,2,3) ((1) Cardiff University, (2) "
- "JPL, (3) Caltech, (4) Stanford University, (5) University of Cambridge, (6) "
- "National University of Ireland Maynooth, (7) University of Edinburgh, (8) "
- "Universidade Tecnica de Lisboa, (9) University of Chicago, (10) Laboratoire "
- "APC/CNRS, (11) NASA Goddard, (12) University of Manchester, (13) UCL, (14) "
- "IPAC) "
- )),
- [
- ["QUaD collaboration", "", ""],
- ["Gupta", "S.", "", "Cardiff University"],
- ["Ade", "P.", "", "Cardiff University"],
- ["Bock", "J.", "", "JPL", "Caltech"],
- ["Bowden", "M.", "", "Cardiff University", "Stanford University"],
- ["Brown", "M. L.", "", "University of Cambridge"],
- ["Cahill", "G.", "", "National University of Ireland Maynooth"],
- [
- "Castro", "P. G.", "",
- "University of Edinburgh",
- "Universidade Tecnica de Lisboa"
- ],
- ["Church", "S.", "", "Stanford University"],
- ["Culverhouse", "T.", "", "University of Chicago"],
- ["Friedman", "R. B.", "", "University of Chicago"],
- ["Ganga", "K.", "", "Laboratoire APC/CNRS"],
- ["Gear", "W. K.", "", "Cardiff University"],
- ["Hinderks", "J.", "", "University of Cambridge", "NASA Goddard"],
- ["Kovac", "J.", "", "Caltech"],
- ["Lange", "A. E.", "", "Stanford University"],
- ["Leitch", "E.", "", "JPL", "Caltech"],
- ["Melhuish", "S. J.", "", "University of Manchester"],
- ["Memari", "Y.", "", "University of Edinburgh"],
- ["Murphy", "J. A.", "", "National University of Ireland Maynooth"],
- ["Orlando", "A.", "", "Cardiff University", "Caltech"],
- ["O'Sullivan", "C.", "", "National University of Ireland Maynooth"],
- ["Piccirillo", "L.", "", "University of Manchester"],
- ["Pryke", "C.", "", "University of Chicago"],
- ["Rajguru", "N.", "", "Cardiff University", "UCL"],
- ["Rusholme", "B.", "", "Stanford University", "IPAC"],
- ["Schwarz", "R.", "", "University of Chicago"],
- ["Taylor", "A. N.", "", "University of Edinburgh"],
- ["Thompson", "K. L.", "", "Stanford University"],
- ["Turner", "A. H.", "", "Cardiff University"],
- ["Wu", "E. Y. S.", "", "Stanford University"],
- ["Zemcov", "M.", "", "Cardiff University", "JPL", "Caltech"]
- ],
- 'parse_author_affil (mind-blowing) 1/1'
- )
-
- # Problem case with 1110.4366
- self.assertListEqual(
- parse_author_affil(
- 'Matthew Everitt, Robert M. Heath and Viv Kendon'),
- [['Everitt', 'Matthew', ''],
- ['Heath', 'Robert M.', ''],
- ['Kendon', 'Viv', '']],
- 'parse_author_affil for 1110.4366'
- )
-
- # look like bugs, but aren't
- self.assertListEqual(
- parse_author_affil('sum won ((lab a), (lab b))'),
- [['won', 'sum', '']],
- 'parse_author_affil (bug imposter) 1/2'
- )
- self.assertListEqual(
- parse_author_affil('sum won ((lab a) (lab b))'),
- [['won', 'sum', '']],
- 'parse_author_affil (bug imposter) 2/2'
- )
-
- self.assertListEqual(
- parse_author_affil('Anatoly Zlotnik and Jr-Shin Li'),
- [['Zlotnik', 'Anatoly', ''],
- ['Li', 'Jr-Shin', '']],
- 'jr issue (Anatoly Zlotnik and Jr-Shin Li)'
- )
-
- # ====== Extra tests for arXiv::AuthorAffil ARXIVDEV-728 ======
-
- # [parse_author_affil]
- self.assertListEqual(
- parse_author_affil(''),
- [],
- 'parse_author_affil (empty)'
- )
- self.assertListEqual(
- parse_author_affil('Simeon Warner Jr'),
- [['Warner', 'Simeon', 'Jr']],
- 'parse_author_affil (basic) 2/12'
- )
- self.assertListEqual(
- parse_author_affil('BELLE Collaboration'),
- [['BELLE Collaboration', '', '']],
- 'parse_author_affil (lone "BELLE Collaboration") 2/3'
- )
-
- self.assertListEqual(
- parse_author_affil('BELLE Collaboration'),
- [['BELLE Collaboration', '', '']],
- 'parse_author_affil (lone "BELLE Collaboration") 2/3'
- )
-
- def test_collaboration_at_front(self):
- f1 = path_of_for_test('data/abs_files/ftp/arxiv/papers/0808/0808.4142.abs')
- meta = AbsMetaSession.parse_abs_file(filename=f1)
- paflst = parse_author_affil(meta.authors.raw)
- self.assertListEqual(paflst, [['D0 Collaboration', '', ''], ['Abazov', 'V.', '']])
diff --git a/tests/test_browse.py b/tests/test_browse.py
index 2c6cc8098..d14bc12cb 100644
--- a/tests/test_browse.py
+++ b/tests/test_browse.py
@@ -1,8 +1,9 @@
import unittest
from bs4 import BeautifulSoup
-
from tests.test_abs_parser import ABS_FILES
+
+from arxiv import taxonomy
from browse.services.document.metadata import AbsMetaSession
from browse.domain.license import ASSUMED_LICENSE_URI
@@ -18,6 +19,104 @@ def setUp(self):
app.config['APPLICATION_ROOT'] = ''
self.app = app.test_client()
+ def test_home(self):
+ """Test the home page."""
+ rv = self.app.get('/')
+ self.assertEqual(rv.status_code, 200)
+ html = BeautifulSoup(rv.data.decode('utf-8'), 'html.parser')
+
+ for group_key, group_value in taxonomy.definitions.GROUPS.items():
+ if group_key == 'grp_test':
+ continue
+ auths_elmt = html.find('h2', text=group_value['name'])
+ self.assertTrue(auths_elmt, f"{group_value['name']} in h2 element")
+ self.assertFalse(html.find('h2', text='Test'),
+ "'Test' group should not be shown on homepage")
+
+ def test_tb(self):
+ """Test the /tb/ page."""
+ rv = self.app.get('/tb/1901.99999')
+ self.assertEqual(rv.status_code, 404)
+
+ rv = self.app.get('/tb/')
+ self.assertEqual(rv.status_code, 404)
+
+ rv = self.app.get('/tb/foo')
+ self.assertEqual(rv.status_code, 404)
+
+ rv = self.app.get('/tb/0808.4142')
+ self.assertEqual(rv.status_code, 200)
+
+ html = BeautifulSoup(rv.data.decode('utf-8'), 'html.parser')
+ h2_elmt = html.find('h2')
+ h2_txt = h2_elmt.get_text()
+ self.assertTrue(h2_elmt, 'Should have element')
+ self.assertEquals(h2_txt, 'Trackbacks for 0808.4142')
+ tb_a_tags = html.find_all('a', 'mathjax', rel='external nofollow')
+ self.assertGreater(len(tb_a_tags), 1,
+ 'There should be more than one tag for trackbacks')
+ h1_elmt = html.find('div', id='abs')
+ h1_txt = h1_elmt.get_text()
+ self.assertTrue(h1_elmt, 'Should have element')
+ self.assertRegex(
+ h1_txt,
+ r'Observation of the doubly strange b baryon Omega_b-',
+ ' element contains title of article')
+
+ def test_tb_recent(self):
+ """Test the /tb/recent page."""
+ rv = self.app.get('/tb/recent')
+ self.assertEqual(rv.status_code, 200)
+
+ rv = self.app.post('/tb/recent', data=dict(views='50'))
+ self.assertEqual(rv.status_code, 200, 'POST with integer OK')
+
+ rv = self.app.post('/tb/recent', data=dict(views='bar'))
+ self.assertEqual(rv.status_code, 400, 'POST with non-integer not OK')
+
+ rv = self.app.get('/tb/recent/foo')
+ self.assertEqual(rv.status_code, 404)
+
+ rv = self.app.post('/tb/recent', data=dict(views='1'))
+ self.assertEqual(rv.status_code, 200, 'POST with views==1 OK')
+ html = BeautifulSoup(rv.data.decode('utf-8'), 'html.parser')
+ tb_a_tags = html.find_all('a', 'mathjax', rel='external nofollow')
+ self.assertEquals(len(tb_a_tags), 1,
+ 'There should be exactly one trackback link')
+
+ def test_stats_today(self):
+ """Test the /stats/today page."""
+ rv = self.app.get('/stats/today')
+ self.assertEqual(rv.status_code, 200)
+ rv = self.app.get('/stats/today?date=20190102')
+ self.assertEqual(rv.status_code, 200)
+ html = BeautifulSoup(rv.data.decode('utf-8'), 'html.parser')
+
+ csv_dl_elmt = html.find('a', {'href': '/stats/get_hourly?date=20190102'})
+ self.assertIsNotNone(csv_dl_elmt,
+ 'csv download link exists')
+
+ def test_stats_monthly_downloads(self):
+ """Test the /stats/monthly_downloads page."""
+ rv = self.app.get('/stats/monthly_downloads')
+ self.assertEqual(rv.status_code, 200)
+ html = BeautifulSoup(rv.data.decode('utf-8'), 'html.parser')
+
+ csv_dl_elmt = html.find('a', {'href': '/stats/get_monthly_downloads'})
+ self.assertIsNotNone(csv_dl_elmt,
+ 'csv download link exists')
+
+ def test_stats_monthly_submissions(self):
+ """Test the /stats/monthly_submissions page."""
+ rv = self.app.get('/stats/monthly_submissions')
+ self.assertEqual(rv.status_code, 200)
+ html = BeautifulSoup(rv.data.decode('utf-8'), 'html.parser')
+
+ csv_dl_elmt = html.find('a', {'href': '/stats/get_monthly_submissions'})
+ self.assertIsNotNone(csv_dl_elmt,
+ 'csv download link exists')
+
+
def test_abs_without_license_field(self):
f1 = ABS_FILES + '/ftp/arxiv/papers/0704/0704.0001.abs'
m = AbsMetaSession.parse_abs_file(filename=f1)
@@ -148,7 +247,8 @@ def test_requested_version(self):
self.assertIsNotNone(pdf_dl_elmt,
'pdf download link without version affix exists')
pdf_dl_elmt = html.find('a', {'href': '/pdf/physics/9707012v'})
- self.assertIsNone(pdf_dl_elmt, 'pdf download link with version affix does not exist')
+ self.assertIsNone(
+ pdf_dl_elmt, 'pdf download link with version affix does not exist')
rv = self.app.get('/abs/physics/9707012v4')
self.assertEqual(rv.status_code, 200)
@@ -200,7 +300,6 @@ def test_1501_9999(self):
'href="ftp://ftp.arxiv.org/cheese.txt"' in rv.data.decode('utf-8'),
"FTP URLs should be turned into links ARXIVNG-1242")
-
def test_160408245(self):
"""Test linking in 1604.08245."""
id = '1604.08245'
@@ -234,62 +333,65 @@ def test_arxivng_1246(self):
def test_authors_and_arxivId_in_title(self):
id = '1501.99999'
- rv = self.app.get('/abs/'+id)
+ rv = self.app.get('/abs/' + id)
self.assertEqual(rv.status_code, 200)
html = BeautifulSoup(rv.data.decode('utf-8'), 'html.parser')
- title_elmt = html.find('h1','title')
- self.assertTrue(title_elmt,'Should title element')
+ title_elmt = html.find('h1', 'title')
+ self.assertTrue(title_elmt, 'Should title element')
ida = title_elmt.find('a')
self.assertTrue(ida, 'Should be tag in title')
- self.assertIsNotNone(ida['href'],' tag in title should have href')
- self.assertEqual(ida['href'], '/abs/1501.99998')
+ self.assertIsNotNone(ida['href'], ' tag in title should have href')
+ self.assertEqual(ida['href'], 'https://arxiv.org/abs/1501.99998')
+
self.assertEqual(ida.text, '1501.99998')
- au_a_tags = html.find('div','authors').find_all('a')
- self.assertGreater(len(au_a_tags), 1, 'Should be some a tags for authors')
+ au_a_tags = html.find('div', 'authors').find_all('a')
+ self.assertGreater(len(au_a_tags), 1,
+ 'Should be some a tags for authors')
self.assertNotIn('query=The', au_a_tags[0]['href'],
'Collaboration author query should not have "The"')
self.assertEqual(au_a_tags[0].text, 'SuperSuper Collaboration')
-
def test_long_author_colab(self):
id = '1501.05201'
- rv = self.app.get('/abs/'+id)
+ rv = self.app.get('/abs/' + id)
self.assertEqual(rv.status_code, 200)
html = BeautifulSoup(rv.data.decode('utf-8'), 'html.parser')
- auths_elmt = html.find('div','authors')
- self.assertTrue(auths_elmt,'Should authors div element')
+ auths_elmt = html.find('div', 'authors')
+ self.assertTrue(auths_elmt, 'Should authors div element')
a_tags = auths_elmt.find_all('a')
- self.assertEqual(len(a_tags), 2, 'Should be two tags in authors div')
+ self.assertEqual(
+ len(a_tags), 2, 'Should be two tags in authors div')
- colab=a_tags[1]
+ colab = a_tags[1]
- self.assertIsNotNone(colab['href'],' tag in title should have href')
- self.assertEqual(colab['href'], 'https://arxiv.org/search/physics?searchtype=author&query=ILL%2FESS%2FLiU+collaboration')
- self.assertEqual(colab.text, 'ILL/ESS/LiU collaboration for the development of the B10 detector technology in the framework of the CRISP project')
+ self.assertIsNotNone(
+ colab['href'], ' tag in title should have href')
+ self.assertEqual(
+ colab['href'], 'https://arxiv.org/search/physics?searchtype=author&query=ILL%2FESS%2FLiU+collaboration')
+ self.assertEqual(
+ colab.text, 'ILL/ESS/LiU collaboration for the development of the B10 detector technology in the framework of the CRISP project')
-
- @unittest.skip("In current implementation, conflicts with comma test below.")
+ @unittest.skip("In current implementation, conflicts with comma test below.")
def test_space_in_author_list(self):
id = '1210.8438'
- rv = self.app.get('/abs/'+id)
+ rv = self.app.get('/abs/' + id)
self.assertEqual(rv.status_code, 200)
html = BeautifulSoup(rv.data.decode('utf-8'), 'html.parser')
- auths_elmt = html.find('div','authors')
- self.assertTrue(auths_elmt,'Should authors div element')
+ auths_elmt = html.find('div', 'authors')
+ self.assertTrue(auths_elmt, 'Should authors div element')
self.assertIn('Zhe (Rita) Liang,', auths_elmt.text,
'Should be a space after (Rita)')
-
def test_comma_in_author_list(self):
id = '0704.0155'
- rv = self.app.get('/abs/'+id)
+ rv = self.app.get('/abs/' + id)
self.assertEqual(rv.status_code, 200)
html = BeautifulSoup(rv.data.decode('utf-8'), 'html.parser')
auths_elmt = html.find('div', 'authors')
@@ -313,5 +415,34 @@ def test_psi_in_abs(self):
self.assertIn('The phase difference $\phi$, between the superconducting',
abs_elmt.text,
"Expecting uncoverted $\phi$ in html abstract.")
-
+ def test_year(self):
+ rv = self.app.get('/year/astro-ph/09')
+ self.assertEqual(rv.status_code, 200)
+
+ rv = self.app.get('/year/astro-ph/')
+ self.assertEqual( rv.status_code, 200)
+
+ rv = self.app.get('/year/astro-ph')
+ self.assertEqual( rv.status_code, 200)
+
+ rv = self.app.get('/year/astro-ph/09/')
+ self.assertEqual(rv.status_code, 200)
+
+ rv = self.app.get('/year')
+ self.assertEqual( rv.status_code, 404)
+
+ rv = self.app.get('/year/astro-ph/9999')
+ self.assertEqual(rv.status_code, 307, 'Future year should cause temporary redirect')
+
+ rv = self.app.get('/year/fakearchive/01')
+ self.assertNotEqual(rv.status_code, 200)
+ self.assertLess( rv.status_code, 500, 'should not cause a 5XX')
+
+ rv = self.app.get('/year/002/0000')
+ self.assertLess( rv.status_code, 500, 'should not cause a 5XX')
+
+ rv = self.app.get('/year/astro-py/9223372036854775808')
+ self.assertLess( rv.status_code, 500, 'should not cause a 5XX')
+
+
diff --git a/tests/test_categories.py b/tests/test_categories.py
index 00a588420..a05f0b99f 100644
--- a/tests/test_categories.py
+++ b/tests/test_categories.py
@@ -3,7 +3,7 @@
from tests.test_abs_parser import ABS_FILES
from browse.services.document.metadata import AbsMetaSession
-
+from browse.domain.category import Category
class CategoriesTest(unittest.TestCase):
@@ -15,7 +15,7 @@ def test_categories_for_0906_3421v1_cats(self):
['Statistical Mechanics (cond-mat.stat-mech)', 'Mathematical Physics (math-ph)'])
doc = self.absService.get_abs(id)
self.assertIsNotNone(doc)
- assert_that(doc.primary_category.display_str(), equal_to(primary))
+ assert_that(doc.primary_category.display, equal_to(primary))
assert_that(doc.display_secondaries(), equal_to(secondaries))
def test_categories_for_0906_3421_cats(self):
@@ -23,7 +23,7 @@ def test_categories_for_0906_3421_cats(self):
['Statistical Mechanics (cond-mat.stat-mech)', 'Mathematical Physics (math-ph)'])
doc = self.absService.get_abs(id)
self.assertIsNotNone(doc)
- assert_that(doc.primary_category.display_str(), equal_to(primary))
+ assert_that(doc.primary_category.display, equal_to(primary))
assert_that(doc.display_secondaries(), equal_to(secondaries))
def test_categories_for_0704_0129_cats(self):
@@ -31,7 +31,7 @@ def test_categories_for_0704_0129_cats(self):
['Mathematical Physics (math-ph)', 'Analysis of PDEs (math.AP)'])
doc = self.absService.get_abs(id)
self.assertIsNotNone(doc)
- assert_that(doc.primary_category.display_str(), equal_to(primary))
+ assert_that(doc.primary_category.display, equal_to(primary))
assert_that(doc.display_secondaries(), equal_to(secondaries))
def test_categories_for_0704_0914_cats(self):
@@ -39,7 +39,7 @@ def test_categories_for_0704_0914_cats(self):
['Mathematical Physics (math-ph)', 'Optics (physics.optics)'])
doc = self.absService.get_abs(id)
self.assertIsNotNone(doc)
- assert_that(doc.primary_category.display_str(), equal_to(primary))
+ assert_that(doc.primary_category.display, equal_to(primary))
assert_that(doc.display_secondaries(), equal_to(secondaries))
def test_categories_for_0704_0582_cats(self):
@@ -47,7 +47,7 @@ def test_categories_for_0704_0582_cats(self):
'Probability (math.PR)', ['Mathematical Physics (math-ph)'])
doc = self.absService.get_abs(id)
self.assertIsNotNone(doc)
- assert_that(doc.primary_category.display_str(), equal_to(primary))
+ assert_that(doc.primary_category.display, equal_to(primary))
assert_that(doc.display_secondaries(), equal_to(secondaries))
def test_categories_for_0704_0495_cats(self):
@@ -55,7 +55,7 @@ def test_categories_for_0704_0495_cats(self):
['Mathematical Physics (math-ph)'])
doc = self.absService.get_abs(id)
self.assertIsNotNone(doc)
- assert_that(doc.primary_category.display_str(), equal_to(primary))
+ assert_that(doc.primary_category.display, equal_to(primary))
assert_that(doc.display_secondaries(), equal_to(secondaries))
def test_categories_for_0704_0681_cats(self):
@@ -63,7 +63,7 @@ def test_categories_for_0704_0681_cats(self):
['Statistical Mechanics (cond-mat.stat-mech)', 'Instrumentation and Detectors (physics.ins-det)', 'Optics (physics.optics)'])
doc = self.absService.get_abs(id)
self.assertIsNotNone(doc)
- assert_that(doc.primary_category.display_str(), equal_to(primary))
+ assert_that(doc.primary_category.display, equal_to(primary))
assert_that(doc.display_secondaries(), equal_to(secondaries))
def test_categories_for_0704_0761_cats(self):
@@ -71,7 +71,7 @@ def test_categories_for_0704_0761_cats(self):
['Soft Condensed Matter (cond-mat.soft)', 'Chemical Physics (physics.chem-ph)'])
doc = self.absService.get_abs(id)
self.assertIsNotNone(doc)
- assert_that(doc.primary_category.display_str(), equal_to(primary))
+ assert_that(doc.primary_category.display, equal_to(primary))
assert_that(doc.display_secondaries(), equal_to(secondaries))
def test_categories_for_0704_0528_cats(self):
@@ -79,7 +79,7 @@ def test_categories_for_0704_0528_cats(self):
['Information Theory (cs.IT)'])
doc = self.absService.get_abs(id)
self.assertIsNotNone(doc)
- assert_that(doc.primary_category.display_str(), equal_to(primary))
+ assert_that(doc.primary_category.display, equal_to(primary))
assert_that(doc.display_secondaries(), equal_to(secondaries))
def test_categories_for_0704_0869_cats(self):
@@ -87,7 +87,7 @@ def test_categories_for_0704_0869_cats(self):
['Mathematical Physics (math-ph)'])
doc = self.absService.get_abs(id)
self.assertIsNotNone(doc)
- assert_that(doc.primary_category.display_str(), equal_to(primary))
+ assert_that(doc.primary_category.display, equal_to(primary))
assert_that(doc.display_secondaries(), equal_to(secondaries))
def test_categories_for_0704_0796_cats(self):
@@ -95,7 +95,7 @@ def test_categories_for_0704_0796_cats(self):
['Astrophysics (astro-ph)', 'Statistical Mechanics (cond-mat.stat-mech)', 'High Energy Physics - Phenomenology (hep-ph)', 'Mathematical Physics (math-ph)'])
doc = self.absService.get_abs(id)
self.assertIsNotNone(doc)
- assert_that(doc.primary_category.display_str(), equal_to(primary))
+ assert_that(doc.primary_category.display, equal_to(primary))
assert_that(doc.display_secondaries(), equal_to(secondaries))
def test_categories_for_0704_0046_cats(self):
@@ -103,7 +103,7 @@ def test_categories_for_0704_0046_cats(self):
'Quantum Physics (quant-ph)', ['Information Theory (cs.IT)'])
doc = self.absService.get_abs(id)
self.assertIsNotNone(doc)
- assert_that(doc.primary_category.display_str(), equal_to(primary))
+ assert_that(doc.primary_category.display, equal_to(primary))
assert_that(doc.display_secondaries(), equal_to(secondaries))
def test_categories_for_0704_0976_cats(self):
@@ -112,7 +112,7 @@ def test_categories_for_0704_0976_cats(self):
'Chaotic Dynamics (nlin.CD)'])
doc = self.absService.get_abs(id)
self.assertIsNotNone(doc)
- assert_that(doc.primary_category.display_str(), equal_to(primary))
+ assert_that(doc.primary_category.display, equal_to(primary))
assert_that(doc.display_secondaries(), equal_to(secondaries))
def test_categories_for_0704_0041_cats(self):
@@ -120,7 +120,7 @@ def test_categories_for_0704_0041_cats(self):
['Mathematical Physics (math-ph)'])
doc = self.absService.get_abs(id)
self.assertIsNotNone(doc)
- assert_that(doc.primary_category.display_str(), equal_to(primary))
+ assert_that(doc.primary_category.display, equal_to(primary))
assert_that(doc.display_secondaries(), equal_to(secondaries))
def test_categories_for_0704_0918_cats(self):
@@ -128,7 +128,7 @@ def test_categories_for_0704_0918_cats(self):
'Algebraic Geometry (math.AG)', ['Statistics Theory (math.ST)'])
doc = self.absService.get_abs(id)
self.assertIsNotNone(doc)
- assert_that(doc.primary_category.display_str(), equal_to(primary))
+ assert_that(doc.primary_category.display, equal_to(primary))
assert_that(doc.display_secondaries(), equal_to(secondaries))
def test_categories_for_0704_0123_cats(self):
@@ -136,7 +136,7 @@ def test_categories_for_0704_0123_cats(self):
['Other Condensed Matter (cond-mat.other)', 'Optics (physics.optics)'])
doc = self.absService.get_abs(id)
self.assertIsNotNone(doc)
- assert_that(doc.primary_category.display_str(), equal_to(primary))
+ assert_that(doc.primary_category.display, equal_to(primary))
assert_that(doc.display_secondaries(), equal_to(secondaries))
def test_categories_for_0704_0520_cats(self):
@@ -145,7 +145,7 @@ def test_categories_for_0704_0520_cats(self):
'Atomic Physics (physics.atom-ph)'])
doc = self.absService.get_abs(id)
self.assertIsNotNone(doc)
- assert_that(doc.primary_category.display_str(), equal_to(primary))
+ assert_that(doc.primary_category.display, equal_to(primary))
assert_that(doc.display_secondaries(), equal_to(secondaries))
def test_categories_for_0704_0084_cats(self):
@@ -154,7 +154,7 @@ def test_categories_for_0704_0084_cats(self):
'Fluid Dynamics (physics.flu-dyn)'])
doc = self.absService.get_abs(id)
self.assertIsNotNone(doc)
- assert_that(doc.primary_category.display_str(), equal_to(primary))
+ assert_that(doc.primary_category.display, equal_to(primary))
assert_that(doc.display_secondaries(), equal_to(secondaries))
def test_categories_for_0704_0588_cats(self):
@@ -162,7 +162,7 @@ def test_categories_for_0704_0588_cats(self):
'Probability (math.PR)', ['Statistics Theory (math.ST)'])
doc = self.absService.get_abs(id)
self.assertIsNotNone(doc)
- assert_that(doc.primary_category.display_str(), equal_to(primary))
+ assert_that(doc.primary_category.display, equal_to(primary))
assert_that(doc.display_secondaries(), equal_to(secondaries))
def test_categories_for_0704_0687_cats(self):
@@ -170,5 +170,17 @@ def test_categories_for_0704_0687_cats(self):
['Mathematical Physics (math-ph)'])
doc = self.absService.get_abs(id)
self.assertIsNotNone(doc)
- assert_that(doc.primary_category.display_str(), equal_to(primary))
+ assert_that(doc.primary_category.display, equal_to(primary))
assert_that(doc.display_secondaries(), equal_to(secondaries))
+
+
+ def test_non_tax_cats(self):
+ cat = Category('nontx.AL')
+ assert_that(cat.display , equal_to('nontx.AL'))
+
+ # TODO: confirm actual desired behavior
+ # cat = Category('math.XX')
+ # assert_that(cat.display , equal_to('Mathematics (math.XX)'))
+
+ cat = Category('notaxArc')
+ assert_that(cat.display , equal_to('notaxArc'))
diff --git a/tests/test_click_through.py b/tests/test_click_through.py
deleted file mode 100644
index a6a6fd3f5..000000000
--- a/tests/test_click_through.py
+++ /dev/null
@@ -1,15 +0,0 @@
-# from unittest import TestCase
-import unittest
-
-from hypothesis import given
-from hypothesis.strategies import text
-
-from browse.util.clickthrough import create_hash, is_hash_valid
-
-
-class TestClickthrough(unittest.TestCase):
-
- @given(text(), text())
- def test_clickthrough(self, s, v):
- h = create_hash(s, v)
- self.assertTrue(is_hash_valid(s, v, h))
diff --git a/tests/test_cookies.py b/tests/test_cookies.py
new file mode 100644
index 000000000..db1b8b22b
--- /dev/null
+++ b/tests/test_cookies.py
@@ -0,0 +1,37 @@
+"""test cookies"""
+import unittest
+
+#from bs4 import BeautifulSoup
+
+from app import app
+
+
+class CookiesPageTest(unittest.TestCase):
+
+ def setUp(self):
+ app.testing = True
+ app.config['APPLICATION_ROOT'] = ''
+ self.app = app.test_client()
+
+
+ def test_cookies_with_no_params(self):
+ """Test the cookies page."""
+ rv = self.app.get('/cookies')
+ self.assertEqual(rv.status_code, 200)
+ html = rv.data.decode('utf-8')
+ self.assertIn('Select preferred download format', html)
+ self.assertIn('show additional debugging information', html, 'should have SHOW debugging link')
+
+ def test_cookies_with_debug(self):
+ """Test the cookies page."""
+ rv = self.app.get('/cookies?debug=1')
+ self.assertEqual(rv.status_code, 200)
+ html = rv.data.decode('utf-8')
+ self.assertIn('Select preferred download format', html)
+ self.assertIn('hide debugging information', html, 'should have HIDE debugging link')
+
+ def test_post_to_cookies(self):
+ rv = self.app.post('/cookies/set?debug=1', data={'ps':'pdf'})
+ self.assertEqual(rv.status_code, 302)
+ cookies = map(lambda kv: kv[1], filter(lambda kv : kv[0]=='Set-Cookie', rv.headers.items()))
+ self.assertIn('xxx-ps-defaults=pdf; Path=/', cookies)
diff --git a/tests/test_database_service.py b/tests/test_database_service.py
index 025608bda..f6d72e62f 100644
--- a/tests/test_database_service.py
+++ b/tests/test_database_service.py
@@ -4,9 +4,11 @@
from unittest import mock, TestCase
from unittest.mock import Mock, patch
from sqlalchemy.exc import SQLAlchemyError
-
from sqlalchemy.orm.exc import NoResultFound
+from browse.services.database.models import TrackbackPing
+from browse.domain.identifier import Identifier
+
from tests import grep_f_count, execute_sql_files, path_of_for_test
@@ -82,6 +84,10 @@ def setUpClass(cls) -> None:
execute_sql_files(sql_files, database.db.engine)
database.db.session.commit()
+ """Disable logging to avoid messy output during testing"""
+ import logging
+ logging.disable(logging.WARNING)
+
def test_get_institution_returns_a_label(self) -> None:
"""If IP address matches an institution, a label is returned."""
label = TestBrowseDatabaseService.database_service.get_institution(
@@ -123,7 +129,8 @@ def test_get_institution_returns_a_label(self) -> None:
def test_all_trackback_pings(self) -> None:
"""Test if all trackback pings are counted."""
- doc_sql_file = path_of_for_test('data/db/sql/arXiv_trackback_pings.sql')
+ doc_sql_file = path_of_for_test(
+ 'data/db/sql/arXiv_trackback_pings.sql')
count_from_file = grep_f_count(
doc_sql_file,
@@ -151,14 +158,14 @@ def test_all_trackback_pings(self) -> None:
)
def test_trackback_pings(self) -> None:
- """Test if trackback pings for specific paper are counted."""
+ """Test if trackback pings for a specific paper are counted."""
test_paper_id = '0808.4142'
count_from_db: int = TestBrowseDatabaseService.database_service\
.count_trackback_pings(test_paper_id)
count_from_db_list: int = TestBrowseDatabaseService.database_service\
- .get_trackback_pings(test_paper_id).__len__()
+ .get_paper_trackback_pings(test_paper_id).__len__()
self.assertEqual(
- count_from_db, 8,
+ count_from_db, 9,
f'Correct count of pings returned for paper {test_paper_id}'
)
self.assertEqual(
@@ -166,6 +173,21 @@ def test_trackback_pings(self) -> None:
f'Correct count of pings returned for paper {test_paper_id}'
)
+ def test_recent_trackback_pings(self) -> None:
+ """Test if recent trackbacks can be retrieved."""
+ tbs: List = TestBrowseDatabaseService.database_service.\
+ get_recent_trackback_pings(max_trackbacks=-1)
+ self.assertEqual(len(tbs), 0, 'List should be empty')
+ tbs: List = TestBrowseDatabaseService.database_service.\
+ get_recent_trackback_pings(max_trackbacks=25)
+ self.assertGreater(len(tbs), 0, 'List should be nonempty')
+ for tb in tbs:
+ self.assertIsInstance(tb[0], TrackbackPing)
+ self.assertIsInstance(tb[1], str)
+ self.assertIsInstance(Identifier(
+ arxiv_id=tb[1]), Identifier, 'Value looks like an Identifier')
+ self.assertIsInstance(tb[2], str)
+
def test_sciencewise_ping(self) -> None:
"""Test whether paper with version suffix has a ScienceWISE ping."""
test_paper_id_v = '1605.09669v2'
@@ -209,15 +231,23 @@ def test_get_dblp_authors(self) -> None:
TestBrowseDatabaseService.database_service.get_dblp_authors(
test_paper_id), [])
+ def test_get_document_count(self) -> None:
+ """Test document count function."""
+ self.assertGreater(
+ TestBrowseDatabaseService.database_service.get_document_count(),
+ 0,
+ 'There is at least one document in the DB.'
+ )
+
@mock.patch('browse.services.database.models.db.session.query')
- def test_error_conditions(self, mock_query)->None:
+ def test_error_conditions(self, mock_query) -> None:
mock_query.side_effect = NoResultFound
self.assertEqual(
TestBrowseDatabaseService.database_service.get_institution('10.0.0.1'), None)
self.assertEqual([],
- TestBrowseDatabaseService.database_service.get_all_trackback_pings())
+ TestBrowseDatabaseService.database_service.get_all_trackback_pings())
self.assertListEqual(
- TestBrowseDatabaseService.database_service.get_trackback_pings('0704.0361'), [])
+ TestBrowseDatabaseService.database_service.get_paper_trackback_pings('0704.0361'), [])
self.assertEqual(
TestBrowseDatabaseService.database_service.count_trackback_pings('0704.0361'), 0)
self.assertEqual(
@@ -231,12 +261,18 @@ def test_error_conditions(self, mock_query)->None:
mock_query.side_effect = SQLAlchemyError
self.assertRaises(SQLAlchemyError,
TestBrowseDatabaseService.database_service.get_institution, '10.0.0.1')
- self.assertRaises(SQLAlchemyError, TestBrowseDatabaseService.database_service.get_all_trackback_pings)
- self.assertRaises(SQLAlchemyError, TestBrowseDatabaseService.database_service.get_trackback_pings, 'paperx')
- self.assertRaises(SQLAlchemyError, TestBrowseDatabaseService.database_service.count_all_trackback_pings)
- self.assertRaises(SQLAlchemyError, TestBrowseDatabaseService.database_service.has_sciencewise_ping, 'px')
- self.assertRaises(SQLAlchemyError, TestBrowseDatabaseService.database_service.get_dblp_listing_path, 'px')
- self.assertRaises(SQLAlchemyError, TestBrowseDatabaseService.database_service.get_dblp_authors,'authx')
+ self.assertRaises(
+ SQLAlchemyError, TestBrowseDatabaseService.database_service.get_all_trackback_pings)
+ self.assertRaises(
+ SQLAlchemyError, TestBrowseDatabaseService.database_service.get_paper_trackback_pings, 'paperx')
+ self.assertRaises(
+ SQLAlchemyError, TestBrowseDatabaseService.database_service.count_all_trackback_pings)
+ self.assertRaises(
+ SQLAlchemyError, TestBrowseDatabaseService.database_service.has_sciencewise_ping, 'px')
+ self.assertRaises(
+ SQLAlchemyError, TestBrowseDatabaseService.database_service.get_dblp_listing_path, 'px')
+ self.assertRaises(
+ SQLAlchemyError, TestBrowseDatabaseService.database_service.get_dblp_authors, 'authx')
@classmethod
def tearDownClass(cls) -> None:
diff --git a/tests/test_doi.py b/tests/test_doi.py
index 124106d76..18772845b 100644
--- a/tests/test_doi.py
+++ b/tests/test_doi.py
@@ -13,6 +13,7 @@
from app import app
+@unittest.skip('We will move this test and any required test data to arxiv-base in the near future')
class DoiTest(unittest.TestCase):
def setUp(self):
diff --git a/tests/test_exceptions.py b/tests/test_exceptions.py
index 54186696b..2ddc2e6d0 100644
--- a/tests/test_exceptions.py
+++ b/tests/test_exceptions.py
@@ -19,6 +19,8 @@ def setUp(self):
wlog.disabled = True
self.app = create_web_app()
+ self.app.testing = True
+ self.app.config['APPLICATION_ROOT'] = ''
self.client = self.app.test_client()
def test_404(self):
@@ -54,7 +56,8 @@ def test_500(self, mock_abs):
"""Disable logging to avoid messy output during testing"""
self.app.logger.disabled = True
- response = self.client.get('/abs/1234.5678')
- self.assertEqual(response.status_code,
- status.HTTP_500_INTERNAL_SERVER_ERROR)
- self.assertIn('text/html', response.content_type)
+ with self.assertRaises(AbsException):
+ response = self.client.get('/abs/1234.5678')
+ self.assertEqual(response.status_code,
+ status.HTTP_500_INTERNAL_SERVER_ERROR)
+ self.assertIn('text/html', response.content_type)
diff --git a/tests/test_filters.py b/tests/test_filters.py
index ee38e22e1..31c7d8e00 100644
--- a/tests/test_filters.py
+++ b/tests/test_filters.py
@@ -1,5 +1,4 @@
import unittest
-from hamcrest import *
from functools import partial
from jinja2 import escape, Markup, Environment
@@ -7,229 +6,217 @@
from flask import appcontext_pushed, url_for
from app import app
-from browse.filters import line_feed_to_br, tex_to_utf, entity_to_utf
-from browse.util.id_patterns import do_dois_to_tags, do_dois_id_urls_to_tags,\
- do_id_to_tags
+from arxiv.base.urls import links, urlizer, urlize
+from arxiv.base.filters import abstract_lf_to_br, f_tex2utf
+from browse.filters import entity_to_utf
+@unittest.skip("These test features implemented in arxiv-base so move these tests to arxiv-base")
+class Jinja_Custom_Filters_Test(unittest.TestCase):
+ """Browse jinja filter tests."""
-def _id_to_url(id: str):
- return url_for('browse.abstract', arxiv_id=id)
-
-def arxiv_urlize(txt):
- return do_dois_id_urls_to_tags(_id_to_url,None,txt)
-
-def doi_urls(fn, txt):
- return do_dois_to_tags(fn, txt)
-
-def arxiv_id_urls(txt):
- return do_id_to_tags(_id_to_url, txt)
-
-class Jinja_Custom_Fitlers_Test(unittest.TestCase):
def test_with_jinja(self):
- jenv = Environment(autoescape=True)
- jenv.filters['doi_urls'] = partial(doi_urls, lambda x: x)
- assert_that(
- jenv.from_string(
- '{{"something 10.1103/PhysRevD.76.013009 or other"|doi_urls}}'
- ).render(),
- equal_to(
- 'something 10.1103/PhysRevD.76.013009 or other'
- ))
+ """Basic urlize DOI filter test."""
+ with app.app_context():
+ jenv = Environment(autoescape=True)
+ jenv.filters['urlize'] = urlizer(
+ ['doi']
+ )
+ self.assertEqual(
+ jenv.from_string(
+ '{{"something 10.1103/PhysRevD.76.013009 or other"|urlize}}'
+ ).render(),
+ 'something <a class="link-https link-external" data-doi="10.1103/PhysRevD.76.013009" href="https://arxiv.org/ct?url=https%3A%2F%2Fdx.doi.org%2F10.1103%2FPhysRevD.76.013009&v=d0670bbf" rel="external noopener nofollow">10.1103/PhysRevD.76.013009</a> or other'
+ )
def test_with_jinja_escapes(self):
- jenv = Environment(autoescape=True)
- jenv.filters['doi_urls'] = partial(doi_urls, lambda x: x)
- jenv.filters['arxiv_urlize'] = arxiv_urlize
- assert_that(
- jenv.from_string(
- '{{"something 10.1103/PhysRevD.76.013009 or other"|doi_urls}}'
- ).render(),
- equal_to(
- 'something 10.1103/PhysRevD.76.013009 or other'
- ))
-
- assert_that(
- jenv.from_string(
- '{{" something 10.1103/PhysRevD.76.013009"|arxiv_urlize}}'
- ).render(),
- equal_to(
- '<script>bad junk</script> something 10.1103/PhysRevD.76.013009'
- ))
+ """Test the tex2utf filter with jinja escapes."""
+ with app.app_context():
+ jenv = Environment(autoescape=True)
+ jenv.filters['urlize'] = urlizer(
+ ['arxiv_id', 'doi']
+ )
+
+ # TODO: urlize doesn't seem to return a Markup object?
+ self.assertEqual(
+ jenv.from_string(
+ '{{"something 10.1103/PhysRevD.76.013009 or other"|urlize}}'
+ ).render(),
+ 'something <a class="link-https link-external" data-doi="10.1103/PhysRevD.76.013009" href="https://arxiv.org/ct?url=https%3A%2F%2Fdx.doi.org%2F10.1103%2FPhysRevD.76.013009&v=d0670bbf" rel="external noopener nofollow">10.1103/PhysRevD.76.013009</a> or other'
+ )
+
+ self.assertEqual(
+ jenv.from_string(
+ '{{" something 10.1103/PhysRevD.76.013009"|urlize}}'
+ ).render(),
+ '<script>bad junk</script> something <a class="link-https link-external" data-doi="10.1103/PhysRevD.76.013009" href="https://arxiv.org/ct?url=https%3A%2F%2Fdx.doi.org%2F10.1103%2FPhysRevD.76.013009&v=d0670bbf" rel="external noopener nofollow">10.1103/PhysRevD.76.013009</a>'
+ )
def test_doi_filter(self):
- doi_fn = partial(doi_urls, lambda x: x)
-
- s = ''
- self.assertEqual(doi_fn(s), s)
-
- s = 'some test string 23$6#$5<>&456 http://google.com/notadoi'
- assert_that(doi_fn(s), equal_to(escape(s)))
-
- doi = '10.1103/PhysRevD.76.013009'
- doiurl = doi_fn(doi)
- self.assertRegex(doiurl, r'^10.1103/PhysRevD.76.013009
')))
-
- s = f'something something {doi} endthing'
- doiurl = doi_fn(s)
- self.assertRegex(doiurl, r'10.1103/PhysRevA.99.013009'
- ' 10.1103/PhysRevZ.44.023009'
- ' 10.1103/PhysRevX.90.012309'
- ' 10.1103/BioRevX.44.123456'
- )))
-
- txt = ''
- assert_that(
- doi_fn(f'{doi} {txt}'),
- equal_to(
- Markup(f'10.1103/PhysRevD.76.013009 {escape(txt)}')
- ))
+ """Test the urlizer DOI filter."""
+ with app.app_context():
+ s = 'some test string 23$6#$5<>&456 http://google.com/notadoi'
+ urlize_dois = urlizer(
+ ['doi']
+ )
+ self.assertEqual(urlize_dois(s), str(escape(s)))
+
+ doi = '10.1103/PhysRevD.76.013009'
+ doiurl = urlize_dois(doi)
+ self.assertRegex(doiurl, r'^10.1103/PhysRevD.76.013009
'))
+ )
+
+ s = f'something something {doi} endthing'
+ doiurl = urlize_dois(s)
+ self.assertRegex(
+ doiurl, r'10.1103/PhysRevA.99.013009 '
+ '10.1103/PhysRevZ.44.023009 '
+ '10.1103/PhysRevX.90.012309 '
+ '10.1103/BioRevX.44.123456'
+ )
+ )
+ )
+
+ txt = ''
+ self.assertEqual(
+ urlize_dois(f'{doi} {txt}'),
+ str(Markup(f'10.1103/PhysRevD.76.013009 '))
+ )
def test_arxiv_id_urls_basic(self):
- h = 'sosmooth.org' # Totally bogus setup for testing, at least url_for returns something
+ """Test basic urlize for arXiv identifiers."""
+ # a server name is needed for url_for to return something
+ h = 'arxiv.org'
app.config['SERVER_NAME'] = h
with app.app_context():
- assert_that(arxiv_id_urls(''), equal_to(''))
+ self.assertEqual(urlize('', ['arxiv_id']), '')
s = 'some text 134#%$$%&^^%*^&(()*_)_<>?:;[}}'
- assert_that(
- arxiv_id_urls(s), equal_to(escape(s)),
- 'filers should return marked up text, which means its escaped')
- assert_that(
- arxiv_id_urls('hep-th/9901001'),
- equal_to(
- f'hep-th/9901001',
- ))
- assert_that(
- arxiv_id_urls('hep-th/9901001 hep-th/9901002'),
- equal_to(
- f'hep-th/9901001 hep-th/9901002'
- ))
+ self.assertEqual(urlize(s), str(escape(s)),
+ 'filters should return escaped text')
+ self.assertEqual(
+ urlize('hep-th/9901001'),
+ f'hep-th/9901001',
+ )
+ self.assertEqual(
+ urlize('hep-th/9901001 hep-th/9901002'),
+ f'hep-th/9901001 hep-th/9901002'
+ )
def test_arxiv_id_urls_3(self):
+ """Test more complex cases of urlize for arXiv identifiers."""
h = 'sosmooth.org'
app.config['SERVER_NAME'] = h
with app.app_context():
- assert_that(
- arxiv_id_urls('hep-th/9901002'),
- equal_to(
- f'hep-th/9901002',
- ))
- assert_that(
- arxiv_id_urls('hep-th/9901002\n'),
- equal_to(
- f'hep-th/9901002\n',
- ),
+ self.assertEqual(
+ urlize('hep-th/9901002'),
+ f'hep-th/9901002',
)
- assert_that(
- arxiv_id_urls(
- 'arXiv:dg-ga/9401001 hep-th/9901001 hep-th/9901002'),
- equal_to(
- f'arXiv:dg-ga/9401001 hep-th/9901001 hep-th/9901002',
- ),
+ self.assertEqual(
+ urlize('hep-th/9901002\n'),
+ f'hep-th/9901002\n'
+ )
+ self.assertEqual(
+ urlize('arXiv:dg-ga/9401001 hep-th/9901001 hep-th/9901002'),
+ f'arXiv:dg-ga/9401001 hep-th/9901001 hep-th/9901002'
)
def test_arxiv_id_urls_punct(self):
+ """Test cases of of urlize for arXiv identifiers with punctuation."""
h = 'sosmooth.org'
app.config['SERVER_NAME'] = h
with app.app_context():
- assert_that(
- arxiv_id_urls('hep-th/9901002.'),
- equal_to(
- f'hep-th/9901002.',
- ), 'followed by period')
- assert_that(
- arxiv_id_urls('0702.0003.'),
- equal_to(
- f'0702.0003.', ),
+ self.assertEqual(
+ urlize('hep-th/9901002.'),
+ f'hep-th/9901002.',
+ 'followed by period')
+ self.assertEqual(
+ urlize('0702.0003.'),
+ f'0702.0003.',
'followed by period')
- assert_that(
- arxiv_id_urls('hep-th/9901001,hep-th/9901002'),
- equal_to(
- f'hep-th/9901001,hep-th/9901002'
- ), 'filter_urls_ids_escape (ID linking) 3/7')
- assert_that(
- arxiv_id_urls('0702.0003, something'),
- equal_to(
- f'0702.0003, something',
- ), 'followed by comma')
- assert_that(
- arxiv_id_urls('(0702.0003) something'),
- equal_to(
- f'(0702.0003) something',
- ), 'in parens')
+ self.assertEqual(
+ urlize('hep-th/9901001,hep-th/9901002'),
+ f'hep-th/9901001,hep-th/9901002',
+ 'filter_urls_ids_escape (ID linking) 3/7')
+ self.assertEqual(
+ urlize('0702.0003, something'),
+ f'0702.0003, something',
+ 'followed by comma')
+ self.assertEqual(
+ urlize('(0702.0003) something'),
+ f'(0702.0003) something',
+ 'in parens')
def test_arxiv_id_urls_more(self):
+ """Test urlize for arXiv identifiers that have mixed formatting."""
h = 'sosmooth.org'
app.config['SERVER_NAME'] = h
with app.app_context():
self.assertEqual(
- arxiv_id_urls('arXiv:dg-ga/9401001 hep-th/9901001 0704.0001'),
- f'arXiv:dg-ga/9401001 hep-th/9901001 0704.0001',
- 'filter_urls_ids_escape (ID linking) 5/7')
+ urlize('arXiv:dg-ga/9401001 hep-th/9901001 0704.0001'),
+ f'arXiv:dg-ga/9401001 hep-th/9901001 0704.0001',
+ 'urlize (ID linking) 5/7')
def test_arxiv_id_v(self):
+ """Test urlize for arXiv identifers with version affix."""
h = 'sosmooth.org'
app.config['SERVER_NAME'] = h
with app.app_context():
- assert_that(
- arxiv_id_urls(
- 'arXiv:dg-ga/9401001v12 hep-th/9901001v2 0704.0001v1'),
- equal_to(
- f'arXiv:dg-ga/9401001v12 hep-th/9901001v2 0704.0001v1'
- ), 'arxiv ids with version numbers')
+ self.assertEqual(
+ urlize('arXiv:dg-ga/9401001v12 hep-th/9901001v2 0704.0001v1'),
+ f'arXiv:dg-ga/9401001v12 hep-th/9901001v2 0704.0001v1',
+ 'arxiv ids with version numbers')
+ @unittest.skip("TODO: confirm actual desired behavior")
def test_vixra(self):
+ """Test urlize for identifiers prefixed by viXra."""
h = 'sosmooth.org'
app.config['SERVER_NAME'] = h
with app.app_context():
- assert_that(
- arxiv_id_urls('viXra:0704.0001 viXra:1003.0123'),
- equal_to('viXra:0704.0001 viXra:1003.0123'))
+ self.assertEqual(urlize('viXra:0704.0001 viXra:1003.0123'),
+ 'viXra:0704.0001 viXra:1003.0123')
# this is what was expected in legacy, but it doesn't seem right:
- # assert_that(
- # arxiv_id_urls('vixra:0704.0001'),
- # equal_to(f'vixra:0704.0001'))
+ # self.assertEqual(
+ # urlize('vixra:0704.0001'),
+ # f'vixra:0704.0001')
def test_arxiv_id_urls_escaping(self):
+ """Test proper escaping when urlize applied."""
h = 'sosmooth.org'
app.config['SERVER_NAME'] = h
with app.app_context():
ax_id = 'hep-th/9901002'
user_entered_txt = ' div should be escaped'
- ex_txt = escape(user_entered_txt).__html__()
- assert_that(
- arxiv_id_urls(ax_id + user_entered_txt),
- equal_to(
- f'hep-th/9901002{ex_txt}'
- ),
+ ex_txt = Markup(user_entered_txt)
+ self.assertEqual(
+ urlize(ax_id + user_entered_txt),
+ f'hep-th/9901002{ex_txt}',
'Dealing with user entered text with html that should be escaped for safety'
)
jinja_escaped_txt = Markup(
' div should already be escaped by jinja2')
- assert_that(
- arxiv_id_urls(ax_id + jinja_escaped_txt),
- equal_to(
- f'hep-th/9901002{jinja_escaped_txt}'
- ), 'Dealing with text that has been escaped by Jinja2 already')
+ self.assertEqual(
+ urlize(ax_id + jinja_escaped_txt),
+ f'hep-th/9901002{jinja_escaped_txt}',
+ 'Dealing with text that has been escaped by Jinja2 already')
def test_arxiv_id_jinja_escapes(self):
h = 'sosmooth.org'
@@ -237,224 +224,209 @@ def test_arxiv_id_jinja_escapes(self):
with app.app_context():
jenv = Environment(autoescape=True)
- jenv.filters['arxiv_id_urls'] = arxiv_id_urls
- jenv.filters['arxiv_urlize'] = arxiv_urlize
+ jenv.filters['urlize'] = jenv.filters['urlize'] = urlizer(
+ ['arxiv_id', 'doi', 'url']
+ )
- assert_that(
+ self.assertEqual(
jenv.from_string(
- '{{"something hep-th/9901002 or other"|arxiv_id_urls}}').
+ '{{"something hep-th/9901002 or other"|urlize|safe}}').
render(),
- equal_to(
- f'something hep-th/9901002 or other'
- ))
+ f'something hep-th/9901002 or other'
+ )
- assert_that(
+ self.assertEqual(
jenv.from_string(
- '{{" something 10.1103/PhysRevD.76.013009"|arxiv_urlize}}'
+ '{{" something 10.1103/PhysRevD.76.013009"|urlize|safe}}'
).render(),
- equal_to(
- '<script>bad junk</script> something 10.1103/PhysRevD.76.013009'
- ))
+ ' something 10.1103/PhysRevD.76.013009'
+ )
- assert_that(
+ self.assertEqual(
jenv.from_string(
'{{" http://google.com bla bla '
- 'hep-th/9901002 bla"|arxiv_urlize}}').
+ 'hep-th/9901002 bla"|urlize|safe}}').
render(),
- equal_to(
- '<script>bad junk</script> '
- 'this http URL bla bla '
- f'hep-th/9901002 bla'
- ), 'should not double escape')
+ ' '
+ 'this http URL bla bla '
+ f'hep-th/9901002 bla',
+ 'should not double escape')
def test_line_break(self):
- assert_that(
- line_feed_to_br('blal\n bla'), equal_to('blal\n
bla'))
+ """Test the abstract lf to br tag filter."""
+ self.assertEqual(abstract_lf_to_br('blal\n bla'), 'blal\n
bla')
- assert_that(
- line_feed_to_br('\nblal\n bla'), equal_to('\nblal\n
bla'))
+ self.assertEqual(abstract_lf_to_br(
+ '\nblal\n bla'), '\nblal\n
bla')
- assert_that(
- line_feed_to_br('\n blal\n bla'), equal_to('\n blal\n
bla'),
- 'need to not do
on first line')
- assert_that(
- line_feed_to_br('blal\n\nbla'), equal_to('blal\nbla'),
- 'skip blank lines')
+ self.assertEqual(abstract_lf_to_br('\n blal\n bla'),
+ '\n blal\n
bla',
+ 'need to not do
on first line')
+ self.assertEqual(abstract_lf_to_br('blal\n\nbla'), 'blal\nbla',
+ 'skip blank lines')
def test_line_break_jinja(self):
+ """Test the abstract lf to br tag filter with urlize on arXiv IDs."""
h = 'sosmooth.org'
app.config['SERVER_NAME'] = h
with app.app_context():
jenv = Environment(autoescape=True)
- jenv.filters['arxiv_urlize'] = arxiv_urlize
- jenv.filters['line_break'] = line_feed_to_br
+ jenv.filters['urlize'] = urlize
+ jenv.filters['line_break'] = abstract_lf_to_br
- assert_that(
+ self.assertEqual(
jenv.from_string(
'{{" http://google.com something or \n'
'\n'
'no double \\n'
' should have br\n'
- 'hep-th/9901002 other"|line_break|arxiv_urlize}}'
+ 'hep-th/9901002 other"|line_break|urlize|safe}}'
).render(),
- equal_to(
- '<script>bad junk</script> '
- 'this http URL'
- ' something or \n'
- 'no double \n'
- '
should have br\n'
- 'hep-th/9901002 other'
- ),
+ '<script>bad junk</script> '
+ 'this http URL'
+ ' something or \n'
+ 'no double \n'
+ '
should have br\n'
+ 'hep-th/9901002 other',
'line_break and arxiv_id_urls should work together'
)
- def test_tex_to_utf(self):
+ def test_tex2utf(self):
+ """Test the tex2utf filter."""
h = 'sosmooth.org'
app.config['SERVER_NAME'] = h
with app.app_context():
jenv = Environment(autoescape=True)
- jenv.filters['arxiv_id_urls'] = arxiv_id_urls
- jenv.filters['line_break'] = line_feed_to_br
- jenv.filters['doi_urls'] = partial(doi_urls, lambda x: x)
- jenv.filters['arxiv_urlize'] = arxiv_urlize
- jenv.filters['tex_to_utf'] = tex_to_utf
+ jenv.filters['urlize'] = urlize
+ jenv.filters['line_break'] = abstract_lf_to_br
+ jenv.filters['tex2utf'] = f_tex2utf
- assert_that(
- jenv.from_string('{{""|tex_to_utf|arxiv_id_urls}}').render(),
- equal_to(''))
+ self.assertEqual(
+ jenv.from_string('{{""|tex2utf|urlize|safe}}').render(),
+ ''
+ )
title = jenv.from_string(
- '{{"Finite-Size and Finite-Temperature Effects in the Conformally Invariant O(N) Vector Model for 2 ARXIVNG-1227'
+ 'Finite-Size and Finite-Temperature Effects in the Conformally Invariant O(N) Vector Model for 2<d<4',
+ 'tex2utf and arxiv_id_urls should handle < and > ARXIVNG-1227'
)
- assert_that(tex_to_utf('Lu\\\'i'), equal_to('Luí'))
- assert_that(tex_to_utf(Markup('Lu\\\'i')), equal_to('Luí'))
- assert_that(tex_to_utf(Markup(escape('Lu\\\'i'))), equal_to('Luí'))
+ self.assertEqual(f_tex2utf('Lu\\\'i'), 'Luí')
+ self.assertEqual(f_tex2utf(Markup('Lu\\\'i')), 'Luí')
+ self.assertEqual(f_tex2utf(Markup(escape('Lu\\\'i'))), 'Luí')
def test_entity_to_utf(self):
+ """Test the entity to utf filter."""
h = 'sosmooth.org'
app.config['SERVER_NAME'] = h
with app.app_context():
jenv = Environment(autoescape=True)
- jenv.filters['arxiv_id_urls'] = arxiv_id_urls
- jenv.filters['line_break'] = line_feed_to_br
- jenv.filters['doi_urls'] = partial(doi_urls, lambda x: x)
- jenv.filters['arxiv_urlize'] = arxiv_urlize
- jenv.filters['tex_to_utf'] = tex_to_utf
jenv.filters['entity_to_utf'] = entity_to_utf
- assert_that(
+ self.assertEqual(
jenv.from_string('{{ "Martín"|entity_to_utf }}').render(),
- equal_to('Martín'), 'entity_to_utf should work')
- assert_that(
+ 'Martín', 'entity_to_utf should work')
+ self.assertEqual(
jenv.from_string(
'{{ ""|entity_to_utf }}').render(),
- equal_to('<Martín>'),
+ '<Martín>',
'entity_to_utf should work even with < or >')
def test_arxiv_urlize_no_email_links(self):
+ """Test to ensure email addresses are not turned into links."""
h = 'sosmooth.org'
app.config['SERVER_NAME'] = h
with app.app_context():
jenv = Environment(autoescape=True)
- jenv.filters['arxiv_urlize'] = arxiv_urlize
+ jenv.filters['urlize'] = urlize
- assert_that(
+ self.assertEqual(
jenv.from_string(
- '{{ "bob@example.com"|arxiv_urlize }}').render(),
- equal_to('bob@example.com'),
+ '{{ "bob@example.com"|urlize|safe }}').render(),
+ 'bob@example.com',
'arxiv_urlize should not turn emails into links')
- assert_that(
+ self.assertEqual(
jenv.from_string(
- '{{ ""|arxiv_urlize }}').render(),
- equal_to('<bob@example.com>'),
+ '{{ ""|urlize|safe }}').render(),
+ '<bob@example.com>',
'arxiv_urlize should work even with < or >')
def test_arxiv_urlize(self):
-
-
-
+ """Multiple basic urlize tests."""
h = 'sosmooth.org'
app.config['SERVER_NAME'] = h
with app.app_context():
-
- def do_arxiv_urlize(txt):
- return arxiv_urlize(txt)
-
self.assertEqual(
- do_arxiv_urlize('http://example.com/'),
- 'this http URL',
- 'do_arxiv_urlize (URL linking) 1/6')
+ urlize('http://example.com/'),
+ 'this http URL',
+ 'urlize (URL linking) 1/6')
self.assertEqual(
- do_arxiv_urlize('https://example.com/'),
- 'this https URL',
- 'do_arxiv_urlize (URL linking) 2/6')
+ urlize('https://example.com/'),
+ 'this https URL',
+ 'urlize (URL linking) 2/6')
self.assertEqual(
- do_arxiv_urlize('ftp://example.com/'),
- 'this ftp URL',
- 'do_arxiv_urlize (URL linking) 3/6')
+ urlize('ftp://example.com/'),
+ 'this ftp URL',
+ 'urlize (URL linking) 3/6')
self.assertEqual(
- do_arxiv_urlize('http://example.com/.hep-th/9901001'),
- 'this http URL',
- 'do_arxiv_urlize (URL linking) 4/6')
+ urlize('http://example.com/.hep-th/9901001'),
+ 'this http URL',
+ 'urlize (URL linking) 4/6')
self.assertEqual(
- do_arxiv_urlize(
+ urlize(
'http://projecteuclid.org/euclid.bj/1151525136'
),
- 'this http URL',
- 'do_arxiv_urlize (URL linking) 6/6')
- assert_that(
- do_arxiv_urlize(' Correction to Bernoulli (2006), 12, 551--570 http://projecteuclid.org/euclid.bj/1151525136'),
- equal_to(Markup(' Correction to Bernoulli (2006), 12, 551--570 this http URL')),
- 'do_arxiv_urlize (URL linking) 6/6')
+ 'this http URL',
+ 'urlize (URL linking) 6/6')
+ self.assertEqual(
+ urlize(
+ ' Correction to Bernoulli (2006), 12, 551--570 http://projecteuclid.org/euclid.bj/1151525136'),
+ Markup(' Correction to Bernoulli (2006), 12, 551--570 this http URL'),
+ 'urlize (URL linking) 6/6')
# shouldn't match
self.assertEqual(
- do_arxiv_urlize('2448446.4710(5)'), '2448446.4710(5)',
- 'do_arxiv_urlize (should not match) 1/9')
+ urlize('2448446.4710(5)'), '2448446.4710(5)',
+ 'urlize (should not match) 1/9')
self.assertEqual(
- do_arxiv_urlize('HJD=2450274.4156+/-0.0009'),
+ urlize('HJD=2450274.4156+/-0.0009'),
'HJD=2450274.4156+/-0.0009',
- 'do_arxiv_urlize (should not match) 2/9')
+ 'urlize (should not match) 2/9')
self.assertEqual(
- do_arxiv_urlize('T_min[HJD]=49238.83662(14)+0.146352739(11)E.'),
+ urlize('T_min[HJD]=49238.83662(14)+0.146352739(11)E.'),
'T_min[HJD]=49238.83662(14)+0.146352739(11)E.',
- 'do_arxiv_urlize (should not match) 3/9')
+ 'urlize (should not match) 3/9')
self.assertEqual(
- do_arxiv_urlize('Pspin=1008.3408s'), 'Pspin=1008.3408s',
- 'do_arxiv_urlize (should not match) 4/9')
+ urlize('Pspin=1008.3408s'), 'Pspin=1008.3408s',
+ 'urlize (should not match) 4/9')
self.assertEqual(
- do_arxiv_urlize('2453527.87455^{+0.00085}_{-0.00091}'),
+ urlize('2453527.87455^{+0.00085}_{-0.00091}'),
'2453527.87455^{+0.00085}_{-0.00091}',
- 'do_arxiv_urlize (should not match) 5/9')
- self.assertEqual(
- do_arxiv_urlize('2451435.4353'), '2451435.4353',
- 'do_arxiv_urlize (should not match) 6/9')
- assert_that(
- do_arxiv_urlize('cond-mat/97063007'),
- equal_to('cond-mat/97063007'),
- 'do_arxiv_urlize (should match) 7/9')
-
- assert_that(
- do_arxiv_urlize('[http://onion.com/something-funny-about-arxiv-1234]'),
- equal_to('[this http URL]'))
-
- assert_that(
- do_arxiv_urlize('[http://onion.com/?q=something-funny-about-arxiv.1234]'),
- equal_to('[this http URL]'))
-
- assert_that(
- do_arxiv_urlize('http://onion.com/?q=something funny'),
- equal_to('this http URL funny'),
+ 'urlize (should not match) 5/9')
+ self.assertEqual(
+ urlize('2451435.4353'), '2451435.4353',
+ 'urlize (should not match) 6/9')
+ self.assertEqual(
+ urlize('cond-mat/97063007'),
+ 'cond-mat/97063007',
+ 'urlize (should match) 7/9')
+
+ self.assertEqual(
+ urlize('[http://onion.com/something-funny-about-arxiv-1234]'),
+ '[this http URL]')
+
+ self.assertEqual(
+ urlize('[http://onion.com/?q=something-funny-about-arxiv.1234]'),
+ '[this http URL]')
+
+ self.assertEqual(
+ urlize('http://onion.com/?q=something funny'),
+ 'this http URL funny',
'Spaces CANNOT be expected to be part of URLs')
-
- assert_that(
- do_arxiv_urlize('"http://onion.com/something-funny-about-arxiv-1234"'),
- equal_to(Markup('"this http URL"')),
- 'Should handle URL surrounded by double quotes')
+ self.assertEqual(
+ urlize('"http://onion.com/something-funny-about-arxiv-1234"'),
+ '"this http URL"',
+ 'Should handle URL surrounded by double quotes')
diff --git a/tests/test_id_patterns.py b/tests/test_id_patterns.py
deleted file mode 100644
index b566c95d5..000000000
--- a/tests/test_id_patterns.py
+++ /dev/null
@@ -1,253 +0,0 @@
-import unittest
-from hamcrest import *
-import re
-
-from jinja2 import Markup, escape
-from browse.util.id_patterns import _find_match, Matchable, dois_ids_and_urls, \
- _transform_token, do_dois_id_urls_to_tags
-
-
-class Id_Patterns_Test(unittest.TestCase):
-
- def test_basic(self):
- _find_match([], 'test')
-
- m = _find_match( [Matchable([], re.compile(r'test'))], 'test')
- assert_that(m, is_not(None))
-
- m0 = Matchable([], re.compile(r'test'))
- m1 = Matchable([], re.compile(r'tests'))
-
- m = _find_match( [m0, m1], 'test')
- assert_that(m, is_not(None))
- assert_that(m[1], equal_to(m0))
-
- m = _find_match([m0, m1], 'tests')
- assert_that(m, is_not(None))
- assert_that(m[1], equal_to(m0))
-
- m = _find_match([m1, m0], 'tests')
- assert_that(m, is_not(None))
- assert_that(m[1], equal_to(m1))
-
- def test_arxiv_ids(self):
- def find_match(txt):
- return _find_match(dois_ids_and_urls, txt)
-
- assert_that(find_match('math/9901123'), is_not(None))
- assert_that(find_match('hep-ex/9901123'), is_not(None))
- assert_that(find_match('gr-qc/9901123'), is_not(None))
-
- assert_that(find_match('1202.1234'), is_not(None))
- assert_that(find_match('1202.1234v1'), is_not(None))
- assert_that(find_match('1203.12345'), is_not(None))
- assert_that(find_match('1203.12345v1'), is_not(None))
- assert_that(find_match('1203.12345v12'), is_not(None))
-
- # slightly odd but seen in comments
- assert_that(find_match('hep-ph/1203.12345v12'), is_not(None))
-
- def test_find_match(self):
- def find_match(txt):
- return _find_match(dois_ids_and_urls, txt)
-
- assert_that(find_match('junk'), equal_to(None))
- assert_that(find_match(''), equal_to(None))
- assert_that(find_match(' '), equal_to(None))
-
- assert_that(find_match('doi:10.1002/0470841559.ch1'), is_not(None))
- assert_that(find_match('doi:10.1038/nphys1170'), is_not(None))
-
- assert_that(find_match('http://arxiv.org'), is_not(None))
- assert_that(find_match('http://arxiv.org?something=1'), is_not(None))
- assert_that(find_match(
- 'http://arxiv.org?something=1&anohter=2'), is_not(None))
- assert_that(find_match('"http://arxiv.org"'), is_not(None))
-
- def test_transform_token(self):
- # def doi_id_url_transform_token(tkn,fn):
- # return doi_id_url_transform_token(fn, tkn)
-
- assert_that(
- do_dois_id_urls_to_tags( None, None, ''),
- equal_to(''))
-
- assert_that(
- do_dois_id_urls_to_tags( None, None,
- 'it is fine, chapter 234 see'),
- equal_to(Markup(escape('it is fine, chapter 234 see'))))
-
- assert_that(
- do_dois_id_urls_to_tags( None, None, 'http://arxiv.org'),
- equal_to('this http URL'))
-
- assert_that(
- do_dois_id_urls_to_tags( None, None,
- 'Stuff in the front http://arxiv.org other stuff'),
- equal_to('Stuff in the front this http URL other stuff'))
-
- assert_that(
- do_dois_id_urls_to_tags( None, None, '.http://arxiv.org.'),
- equal_to('.this http URL.'))
-
- assert_that(
- do_dois_id_urls_to_tags( None, None, '"http://arxiv.org"'),
- equal_to(Markup('"this http URL"')))
-
- def test_urlize(self):
- def do_arxiv_urlize(txt):
- return do_dois_id_urls_to_tags(None,None, txt)
-
- assert_that(
- do_arxiv_urlize('http://example.com/'),
- equal_to('this http URL'),
- 'do_arxiv_urlize (URL linking) 1/6')
- assert_that(
- do_arxiv_urlize('https://example.com/'),
- equal_to('this https URL'),
- 'do_arxiv_urlize (URL linking) 2/6')
- assert_that(
- do_arxiv_urlize('ftp://example.com/'),
- equal_to('this ftp URL'),
- 'do_arxiv_urlize (URL linking) 3/6')
-
-
- assert_that(
- do_arxiv_urlize(
- 'http://projecteuclid.org/euclid.bj/1151525136'
- ),
- equal_to('this http URL'),
- 'do_arxiv_urlize (URL linking) 6/6')
- # assert_that(
- # do_arxiv_urlize(
- # ' Correction to Bernoulli (2006), 12, 551--570 http://projecteuclid.org/euclid.bj/1151525136'),
- # equal_to(
- # ' Correction to Bernoulli (2006), 12, 551--570 this http URL'),
- # 'do_arxiv_urlize (URL linking) 6/6')
- # shouldn't match
- assert_that(
- do_arxiv_urlize('2448446.4710(5)'), '2448446.4710(5)',
- equal_to('do_arxiv_urlize (should not match) 1/9'))
- self.assertEqual(
- do_arxiv_urlize('HJD=2450274.4156+/-0.0009'),
- 'HJD=2450274.4156+/-0.0009',
- 'do_arxiv_urlize (should not match) 2/9')
- assert_that(
- do_arxiv_urlize(
- 'T_min[HJD]=49238.83662(14)+0.146352739(11)E.'),
- equal_to('T_min[HJD]=49238.83662(14)+0.146352739(11)E.'),
- 'do_arxiv_urlize (should not match) 3/9')
- assert_that(
- do_arxiv_urlize('Pspin=1008.3408s'), 'Pspin=1008.3408s',
- equal_to('do_arxiv_urlize (should not match) 4/9'))
- assert_that(
- do_arxiv_urlize('2453527.87455^{+0.00085}_{-0.00091}'),
- equal_to('2453527.87455^{+0.00085}_{-0.00091}'),
- 'do_arxiv_urlize (should not match) 5/9')
- assert_that(
- do_arxiv_urlize('2451435.4353'), equal_to('2451435.4353'),
- 'do_arxiv_urlize (should not match) 6/9')
-
-
- assert_that(
- do_arxiv_urlize('cond-mat/97063007'),
- equal_to(
- 'cond-mat/97063007'),
- 'do_arxiv_urlize (should match) 7/9')
-
- assert_that(
- do_arxiv_urlize(
- '[http://onion.com/something-funny-about-arxiv-1234]'),
- equal_to('[this http URL]'))
-
- assert_that(
- do_arxiv_urlize(
- '[http://onion.com/?q=something-funny-about-arxiv.1234]'),
- equal_to('[this http URL]'))
-
- assert_that(
- do_arxiv_urlize('http://onion.com/?q=something funny'),
- equal_to(
- 'this http URL funny'),
- 'Spaces CANNOT be expected to be part of URLs')
-
- assert_that(
- do_arxiv_urlize(
- '"http://onion.com/something-funny-about-arxiv-1234"'),
- equal_to(
- Markup('"this http URL"')),
- 'Should handle URL surrounded by double quotes')
-
- assert_that(
- do_arxiv_urlize('< http://example.com/1<2 ><'),
- equal_to('< this http URL<2 ><'),
- 'do_arxiv_urlize (URL linking) 5/6')
-
- assert_that(
- do_arxiv_urlize('Accepted for publication in A&A. The data will be available via CDS, and can be found "http://atlasgal.mpifr-bonn.mpg.de/cgi-bin/ATLASGAL_FILAMENTS.cgi"'),
- equal_to('Accepted for publication in A&A. The data will be available via CDS, and can be found "this http URL"')
- )
-
- assert_that(
- do_arxiv_urlize('see http://www.tandfonline.com/doi/abs/doi:10.1080/15980316.2013.860928?journalCode=tjid20'),
- equal_to('see this http URL')
- )
-
- assert_that(
- do_arxiv_urlize('http://authors.elsevier.com/a/1TcSd,Ig45ZtO'),
- equal_to('this http URL'))
-
- def category_id_test(self):
- def do_arxiv_urlize(txt):
- return do_dois_id_urls_to_tags(None,None, txt)
-
- assert_that(
- do_arxiv_urlize('version of arXiv.math.GR/0512484 (2011).'),
- equal_to('version of arXiv.math.GR/0512484 (2011).'))
-
- def hosts_tests(self):
- def do_arxiv_urlize(txt):
- return do_dois_id_urls_to_tags(None,None, txt)
-
- assert_that(do_arxiv_urlize('can be downloaded from http://rwcc.bao.ac.cn:8001/swap/NLFFF_DBIE_code/HeHan_NLFFF_JGR.pdf'),
- equal_to("can be downloaded from this http URL"),
- "Should deal with ports correctly")
-
-
- assert_that(do_arxiv_urlize("images is at http://85.20.11.14/hosting/punsly/APJLetter4.2.07/"),
- equal_to('images is at this http URL'),
- "should deal with numeric IP correctly")
-
- def urls_with_plus(self):
- def do_arxiv_urlize(txt):
- return do_dois_id_urls_to_tags(lambda x: x, lambda x:x, txt)
-
- assert_that(do_arxiv_urlize('http://www.fkf.mpg.de/andersen/docs/pub/abstract2004+/pavarini_02.pdf'),
- equal_to("this http URL"),
- "Should deal with plus in URL correctly")
-
- def anchors_with_slash(self):
- def do_arxiv_urlize(txt):
- return do_dois_id_urls_to_tags(lambda x: x, lambda x:x, txt)
-
- assert_that(do_arxiv_urlize('https://dms.sztaki.hu/ecml-pkkd-2016/#/app/privateleaderboard'),
- equal_to(Markup("this https URL")),
- "Should deal with slash in URL anchor correctly")
-
- def ftp_test(self):
- cmt = "7 Pages; ftp://ftp%40micrognu%2Ecom:anon%40anon@ftp.micrognu.com/pnenp/conclusion.pdf"
- def do_arxiv_urlize(txt):
- return do_dois_id_urls_to_tags(lambda x: x, lambda x:x, txt)
-
- assert_that(do_arxiv_urlize(cmt),
- equal_to(Markup('7 Pages; this ftp URL')))
-
-
- def arxiv_prefix_test(self):
-
- def do_arxiv_urlize(txt):
- return do_dois_id_urls_to_tags(lambda x: x, lambda x:x, txt)
-
- cmt = "see arxiv:1201.12345"
- assert_that(do_arxiv_urlize(cmt),
- equal_to(Markup('see arXiv:1201.12345')))
diff --git a/tests/test_list_page.py b/tests/test_list_page.py
new file mode 100644
index 000000000..8ecee2f9c
--- /dev/null
+++ b/tests/test_list_page.py
@@ -0,0 +1,485 @@
+import unittest
+import re
+from hamcrest import *
+from unittest.mock import MagicMock
+
+from bs4 import BeautifulSoup
+
+from tests.test_abs_parser import ABS_FILES
+from browse.services.document.metadata import AbsMetaSession
+from browse.domain.license import ASSUMED_LICENSE_URI
+from browse.services.listing.fake_listings import FakeListingFilesService
+from browse.services.listing import ListingService, get_listing_service
+import os
+
+from app import app
+
+
+class ListPageTest(unittest.TestCase):
+
+ def setUp(self):
+ app.testing = True
+ app.config['APPLICATION_ROOT'] = ''
+ self.app = app
+ self.client = app.test_client()
+
+ def test_basic_lists(self):
+ rv = self.client.get('/list/hep-ph/0901')
+ self.assertEqual(rv.status_code, 200)
+ self.assertNotEqual(rv.headers.get('Expires', None), None)
+
+ rv = self.client.get('/list/hep-ph/09')
+ self.assertEqual(rv.status_code, 200)
+ self.assertNotEqual(rv.headers.get('Expires', None), None)
+
+ rv = self.client.get('/list/hep-ph/new')
+ self.assertEqual(rv.status_code, 200)
+ self.assertNotEqual(rv.headers.get('Expires', None), None)
+
+ rv = self.client.get('/list/hep-ph/current')
+ self.assertEqual(rv.status_code, 200)
+ self.assertNotEqual(rv.headers.get('Expires', None), None)
+
+ rv = self.client.get('/list/hep-ph/pastweek')
+ self.assertEqual(rv.status_code, 200)
+ self.assertNotEqual(rv.headers.get('Expires', None), None)
+
+ rv = self.client.get('/list/hep-ph/recent')
+ self.assertEqual(rv.status_code, 200)
+ self.assertNotEqual(rv.headers.get('Expires', None), None)
+
+ rv = self.client.get('/list/hep-ph/0901?skip=925&show=25')
+ self.assertEqual(rv.status_code, 200)
+ self.assertNotEqual(rv.headers.get('Expires', None), None)
+
+ rv = self.client.get('/list/astro-ph/04')
+ self.assertEqual(rv.status_code, 200)
+ self.assertNotEqual(rv.headers.get('Expires', None), None)
+
+ rv = self.client.get('/list/math/92')
+ self.assertEqual(rv.status_code, 200)
+ self.assertNotEqual(rv.headers.get('Expires', None), None)
+
+ rv = self.client.get('/list/math/9201')
+ self.assertEqual(rv.status_code, 200)
+
+ rv = self.client.get('/list/math/0101')
+ self.assertEqual(rv.status_code, 200)
+
+ rv = self.client.get('/list/math/0102')
+ self.assertEqual(rv.status_code, 200)
+
+ rv = self.client.get('/list/math/0103')
+ self.assertEqual(rv.status_code, 200)
+
+ rv = self.client.get('/list/math/0104')
+ self.assertEqual(rv.status_code, 200)
+
+ rv = self.client.get('/list/math/0105')
+ self.assertEqual(rv.status_code, 200)
+
+ rv = self.client.get('/list/math/0106')
+ self.assertEqual(rv.status_code, 200)
+
+ rv = self.client.get('/list/math/0107')
+ self.assertEqual(rv.status_code, 200)
+
+ rv = self.client.get('/list/math/0108')
+ self.assertEqual(rv.status_code, 200)
+
+ rv = self.client.get('/list/math/0109')
+ self.assertEqual(rv.status_code, 200)
+
+ rv = self.client.get('/list/math/0110')
+ self.assertEqual(rv.status_code, 200)
+
+ rv = self.client.get('/list/math/0111')
+ self.assertEqual(rv.status_code, 200)
+
+ rv = self.client.get('/list/math/0112')
+ self.assertEqual(rv.status_code, 200)
+
+ rv = self.client.get('/list/math/01')
+ self.assertEqual(rv.status_code, 200)
+
+ rv = self.client.get('/list/math/18')
+ self.assertEqual(rv.status_code, 200)
+
+ rv = self.client.get('/list/math/20') # year 2020
+ self.assertEqual(rv.status_code, 200)
+
+ rv = self.client.get('/list/math/30') # year 2030
+ self.assertEqual(rv.status_code, 200)
+
+ rv = self.client.get('/list/math/200101')
+ self.assertEqual(rv.status_code, 200)
+
+ def test_listing_authors(self):
+ rv = self.client.get('/list/hep-ph/0901')
+ self.assertEqual(rv.status_code, 200)
+ au = b'Eqab M. Rabei'
+ assert au in rv.data, f'Simple check for author {au} in response.'
+
+ html = BeautifulSoup(rv.data.decode('utf-8'), 'html.parser')
+
+ auDivs = html.find_all('div', 'list-authors')
+ assert_that(auDivs, not_none())
+ assert_that(len(auDivs), greater_than(
+ 5), 'Should have some .list-author divs')
+
+ first_aus = auDivs[0].find_all('a')
+ assert_that(first_aus, has_length(4),
+ 'expect 4 tags for first artcile "Fractional WKB Approximation"')
+
+ assert_that(first_aus[0].get_text(), equal_to('Eqab M. Rabei'))
+ assert_that(first_aus[1].get_text(),
+ equal_to('Ibrahim M. A. Altarazi'))
+ assert_that(first_aus[2].get_text(), equal_to('Sami I. Muslih'))
+ assert_that(first_aus[3].get_text(), equal_to('Dumitru Baleanu'))
+
+ assert_that(auDivs[0].get_text(), is_not(contains_string(' ,')),
+ 'Should not have a comma with a space in front of it')
+
+ def test_paging_first(self):
+ rv = self.client.get('/list/hep-ph/0901')
+ self.assertEqual(rv.status_code, 200)
+
+ rvdata = rv.data.decode('utf-8')
+ html = BeautifulSoup(rvdata, 'html.parser')
+
+ paging = html.find(id='dlpage').find_all('div')[0]
+ assert_that(paging, not_none())
+ tgs = paging.find_all(['span', 'a'])
+ assert_that(tgs, not_none())
+ assert_that(len(tgs), 6)
+
+ assert_that(tgs[0].name, equal_to('span'))
+ assert_that(tgs[0].get_text(), equal_to('1-25'))
+
+ assert_that(tgs[1].name, equal_to('a'))
+ assert_that(tgs[1].get_text(), equal_to('26-50'))
+
+ assert_that(tgs[2].name, equal_to('a'))
+ assert_that(tgs[2].get_text(), equal_to('51-75'))
+
+ assert_that(tgs[3].name, equal_to('a'))
+ assert_that(tgs[3].get_text(), equal_to('76-100'))
+
+ assert_that(tgs[4].name, equal_to('span'))
+ assert_that(tgs[4].get_text(), equal_to('...'))
+
+ assert_that(tgs[5].name, equal_to('a'))
+ assert_that(tgs[5].get_text(), equal_to('1001-1001'))
+
+ # find the first article index tag
+ first_index_atag = html.find(id='articles').find_all(
+ 'dt')[0].find('a', string=re.compile(r'\[\d*\]'))
+ assert_that(first_index_atag, not_none())
+ assert_that(first_index_atag['name'], equal_to('item1'))
+ assert_that(first_index_atag.string, equal_to('[1]'))
+
+ def test_paging_second(self):
+ rv = self.client.get('/list/hep-ph/0901?skip=25&show=25')
+ self.assertEqual(rv.status_code, 200)
+
+ rvdata = rv.data.decode('utf-8')
+ html = BeautifulSoup(rvdata, 'html.parser')
+
+ paging = html.find(id='dlpage').find_all('div')[0]
+ assert_that(paging, not_none())
+ tgs = paging.find_all(['span', 'a'])
+ assert_that(tgs, not_none())
+ assert_that(len(tgs), 7)
+
+ assert_that(tgs[0].name, equal_to('a'))
+ assert_that(tgs[0].get_text(), equal_to('1-25'))
+
+ assert_that(tgs[1].name, equal_to('span'))
+ assert_that(tgs[1].get_text(), equal_to('26-50'))
+
+ assert_that(tgs[2].name, equal_to('a'))
+ assert_that(tgs[2].get_text(), equal_to('51-75'))
+
+ assert_that(tgs[3].name, equal_to('a'))
+ assert_that(tgs[3].get_text(), equal_to('76-100'))
+
+ assert_that(tgs[4].name, equal_to('a'))
+ assert_that(tgs[4].get_text(), equal_to('101-125'))
+
+ assert_that(tgs[5].name, equal_to('span'))
+ assert_that(tgs[5].get_text(), equal_to('...'))
+
+ assert_that(tgs[6].name, equal_to('a'))
+ assert_that(tgs[6].get_text(), equal_to('1001-1001'))
+
+ # find the first article index tag
+ first_index_atag = html.find(id='articles').find_all(
+ 'dt')[0].find('a', string=re.compile(r'\[\d*\]'))
+ assert_that(first_index_atag, not_none())
+ assert_that(first_index_atag['name'], is_not(
+ 'item1'), 'first item index should not be 1')
+ assert_that(first_index_atag.string, equal_to('[26]'))
+
+ def test_paging_middle(self):
+ rv = self.client.get('/list/hep-ph/0901?skip=175&show=25')
+ self.assertEqual(rv.status_code, 200)
+
+ rvdata = rv.data.decode('utf-8')
+ html = BeautifulSoup(rvdata, 'html.parser')
+
+ paging = html.find(id='dlpage').find_all('div')[0]
+ assert_that(paging, not_none())
+ tgs = paging.find_all(['span', 'a'])
+ assert_that(tgs, not_none())
+ assert_that(len(tgs), 7)
+
+ assert_that(tgs[0].name, equal_to('a'))
+ assert_that(tgs[0].get_text(), equal_to('1-25'))
+
+ assert_that(tgs[1].name, equal_to('span'))
+ assert_that(tgs[1].get_text(), equal_to('...'))
+
+ assert_that(tgs[2].name, equal_to('a'))
+ assert_that(tgs[2].get_text(), equal_to('101-125'))
+
+ assert_that(tgs[3].name, equal_to('a'))
+ assert_that(tgs[3].get_text(), equal_to('126-150'))
+
+ assert_that(tgs[4].name, equal_to('a'))
+ assert_that(tgs[4].get_text(), equal_to('151-175'))
+
+ assert_that(tgs[5].name, equal_to('span'))
+ assert_that(tgs[5].get_text(), equal_to('176-200'))
+
+ assert_that(tgs[6].name, equal_to('a'))
+ assert_that(tgs[6].get_text(), equal_to('201-225'))
+
+ assert_that(tgs[7].name, equal_to('a'))
+ assert_that(tgs[7].get_text(), equal_to('226-250'))
+
+ assert_that(tgs[8].name, equal_to('a'))
+ assert_that(tgs[8].get_text(), equal_to('251-275'))
+
+ assert_that(tgs[9].name, equal_to('span'))
+ assert_that(tgs[9].get_text(), equal_to('...'))
+
+ assert_that(tgs[10].name, equal_to('a'))
+ assert_that(tgs[10].get_text(), equal_to('1001-1001'))
+
+ # find the first article index tag
+ first_index_atag = html.find(id='articles').find_all(
+ 'dt')[0].find('a', string=re.compile(r'\[\d*\]'))
+ assert_that(first_index_atag, not_none())
+ assert_that(first_index_atag['name'], is_not(
+ 'item1'), 'first item index should not be 1')
+ assert_that(first_index_atag.string, equal_to('[176]'))
+
+ def test_paging_last(self):
+ rv = self.client.get('/list/hep-ph/0901?skip=1000&show=25')
+ self.assertEqual(rv.status_code, 200)
+
+ rvdata = rv.data.decode('utf-8')
+ html = BeautifulSoup(rvdata, 'html.parser')
+
+ paging = html.find(id='dlpage').find_all('div')[0]
+ assert_that(paging, not_none())
+ tgs = paging.find_all(['span', 'a'])
+ assert_that(tgs, not_none())
+ assert_that(len(tgs), 7)
+
+ assert_that(tgs[0].name, equal_to('a'))
+ assert_that(tgs[0].get_text(), equal_to('1-25'))
+
+ assert_that(tgs[1].name, equal_to('span'))
+ assert_that(tgs[1].get_text(), equal_to('...'))
+
+ assert_that(tgs[2].name, equal_to('a'))
+ assert_that(tgs[2].get_text(), equal_to('926-950'))
+
+ assert_that(tgs[3].name, equal_to('a'))
+ assert_that(tgs[3].get_text(), equal_to('951-975'))
+
+ assert_that(tgs[4].name, equal_to('a'))
+ assert_that(tgs[4].get_text(), equal_to('976-1000'))
+
+ assert_that(tgs[5].name, equal_to('span'))
+ assert_that(tgs[5].get_text(), equal_to('1001-1001'))
+
+ # find the first article index tag
+ first_index_atag = html.find(id='articles').find_all(
+ 'dt')[0].find('a', string=re.compile(r'\[\d*\]'))
+ assert_that(first_index_atag, not_none())
+ assert_that(first_index_atag['name'], is_not(
+ 'item1'), 'first item index should not be 1')
+ assert_that(first_index_atag.string, equal_to('[1001]'))
+
+ def test_paging_penultimate(self):
+ rv = self.client.get('/list/hep-ph/0901?skip=975&show=25')
+ self.assertEqual(rv.status_code, 200)
+
+ rvdata = rv.data.decode('utf-8')
+ html = BeautifulSoup(rvdata, 'html.parser')
+
+ paging = html.find(id='dlpage').find_all('div')[0]
+ assert_that(paging, not_none())
+ tgs = paging.find_all(['span', 'a'])
+ assert_that(tgs, not_none())
+ assert_that(len(tgs), 7)
+
+ assert_that(tgs[0].name, equal_to('a'))
+ assert_that(tgs[0].get_text(), equal_to('1-25'))
+
+ assert_that(tgs[1].name, equal_to('span'))
+ assert_that(tgs[1].get_text(), equal_to('...'))
+
+ assert_that(tgs[2].name, equal_to('a'))
+ assert_that(tgs[2].get_text(), equal_to('901-925'))
+
+ assert_that(tgs[3].name, equal_to('a'))
+ assert_that(tgs[3].get_text(), equal_to('926-950'))
+
+ assert_that(tgs[4].name, equal_to('a'))
+ assert_that(tgs[4].get_text(), equal_to('951-975'))
+
+ assert_that(tgs[5].name, equal_to('span'))
+ assert_that(tgs[5].get_text(), equal_to('976-1000'))
+
+ assert_that(tgs[6].name, equal_to('a'))
+ assert_that(tgs[6].get_text(), equal_to('1001-1001'))
+
+ # find the first article index tag
+ first_index_atag = html.find(id='articles').find_all(
+ 'dt')[0].find('a', string=re.compile(r'\[\d*\]'))
+ assert_that(first_index_atag, not_none())
+ assert_that(first_index_atag['name'], is_not(
+ 'item1'), 'first item index should not be 1')
+ assert_that(first_index_atag.string, equal_to('[976]'))
+
+ def test_paging_925(self):
+ rv = self.client.get('/list/hep-ph/0901?skip=925&show=25')
+ self.assertEqual(rv.status_code, 200)
+
+ rvdata = rv.data.decode('utf-8')
+ html = BeautifulSoup(rvdata, 'html.parser')
+
+ paging = html.find(id='dlpage').find_all('div')[0]
+ assert_that(paging, not_none())
+ tgs = paging.find_all(['span', 'a'])
+ assert_that(tgs, not_none())
+ assert_that(len(tgs), 7)
+
+ assert_that(tgs[0].name, equal_to('a'))
+ assert_that(tgs[0].get_text(), equal_to('1-25'))
+
+ assert_that(tgs[1].name, equal_to('span'))
+ assert_that(tgs[1].get_text(), equal_to('...'))
+
+ assert_that(tgs[2].name, equal_to('a'))
+ assert_that(tgs[2].get_text(), equal_to('851-875'))
+
+ assert_that(tgs[3].name, equal_to('a'))
+ assert_that(tgs[3].get_text(), equal_to('876-900'))
+
+ assert_that(tgs[4].name, equal_to('a'))
+ assert_that(tgs[4].get_text(), equal_to('901-925'))
+
+ assert_that(tgs[5].name, equal_to('span'))
+ assert_that(tgs[5].get_text(), equal_to('926-950'))
+
+ assert_that(tgs[6].name, equal_to('a'))
+ assert_that(tgs[6].get_text(), equal_to('951-975'))
+
+ assert_that(tgs[7].name, equal_to('a'))
+ assert_that(tgs[7].get_text(), equal_to('976-1000'))
+
+ assert_that(tgs[8].name, equal_to('a'))
+ assert_that(tgs[8].get_text(), equal_to('1001-1001'))
+
+ # find the first article index tag
+ first_index_atag = html.find(id='articles').find_all(
+ 'dt')[0].find('a', string=re.compile(r'\[\d*\]'))
+ assert_that(first_index_atag, not_none())
+ assert_that(first_index_atag['name'], is_not(
+ 'item1'), 'first item index should not be 1')
+ assert_that(first_index_atag.string, equal_to('[926]'))
+
+ def test_odd_requests(self):
+ rv = self.client.get('/list/hep-ph/0901?skip=925&show=1000000')
+ self.assertEqual(rv.status_code, 200)
+
+ rv = self.client.get('/list/hep-ph/bogusTimePeriod')
+ self.assertNotEqual(rv.status_code, 200)
+
+ rv = self.client.get('/list/junkarchive')
+ self.assertNotEqual(rv.status_code, 200)
+
+ rv = self.client.get('/list/ao-si/0901?skip=925&show=25')
+ self.assertNotEqual(rv.status_code, 200)
+
+ rv = self.client.get('/list/math/0100')
+ self.assertNotEqual(rv.status_code, 200)
+
+ rv = self.client.get('/list/math/0113')
+ self.assertNotEqual(rv.status_code, 200)
+
+ rv = self.client.get('/list/math/0199')
+ self.assertNotEqual(rv.status_code, 200)
+
+ rv = self.client.get('/list/math/200199')
+ self.assertNotEqual(rv.status_code, 200)
+
+ rv = self.client.get('/list/math/2')
+ self.assertNotEqual(rv.status_code, 200)
+
+ rv = self.client.get('/list/math/2001999999')
+ self.assertNotEqual(rv.status_code, 200)
+
+ def test_abs_service(self):
+ service = ListingService()
+ assert_that(calling(service.list_articles_by_year).with_args('a', 1, 1, 1, 1),
+ raises(NotImplementedError))
+ assert_that(calling(service.list_articles_by_month).with_args('a', 1, 1, 1, 1),
+ raises(NotImplementedError))
+ assert_that(calling(service.list_new_articles).with_args('a', 1, 1),
+ raises(NotImplementedError))
+ assert_that(calling(service.list_pastweek_articles).with_args('a', 1, 1),
+ raises(NotImplementedError))
+
+ assert_that(service.version(), is_not(None))
+
+ def test_not_modified_from_listing_service(self):
+ with self.app.app_context():
+ flservice = get_listing_service()
+ flservice.list_new_articles = MagicMock(return_value={'not_modified': True,
+ 'expires': 'Wed, 21 Oct 2015 07:28:00 GMT'})
+ rv = self.client.get('/list/hep-ph/new')
+ self.assertEqual(
+ rv.status_code, 304, '/list controller should return 304 when service indicates not-modified')
+
+ flservice.list_pastweek_articles = MagicMock(return_value={'not_modified': True,
+ 'expires': 'Wed, 21 Oct 2015 07:28:00 GMT'})
+ rv = self.client.get('/list/hep-ph/recent')
+ self.assertEqual(
+ rv.status_code, 304, '/list controller should return 304 when service indicates not-modified')
+ rv = self.client.get('/list/hep-ph/pastweek')
+ self.assertEqual(
+ rv.status_code, 304, '/list controller should return 304 when service indicates not-modified')
+
+ flservice.list_articles_by_month = MagicMock(return_value={'not_modified': True,
+ 'expires': 'Wed, 21 Oct 2015 07:28:00 GMT'})
+ rv = self.client.get('/list/hep-ph/1801')
+ self.assertEqual(
+ rv.status_code, 304, '/list controller should return 304 when service indicates not-modified')
+
+ flservice.list_articles_by_year = MagicMock(return_value={'not_modified': True,
+ 'expires': 'Wed, 21 Oct 2015 07:28:00 GMT'})
+ rv = self.client.get('/list/hep-ph/18')
+ self.assertEqual(
+ rv.status_code, 304, '/list controller should return 304 when service indicates not-modified')
+
+ def test_list_called_from_archive(self):
+ rv = self.client.get('/list/?archive=hep-ph&year=08&month=03&submit=Go')
+ self.assertEqual(rv.status_code, 200)
+
+ rv = self.client.get('/list/?archive=hep-ph&year=08&month=all&submit=Go')
+ self.assertEqual(rv.status_code, 200)
diff --git a/tests/test_search_authors.py b/tests/test_search_authors.py
index 8fed019dc..b46be69c1 100644
--- a/tests/test_search_authors.py
+++ b/tests/test_search_authors.py
@@ -3,14 +3,14 @@
from unittest import TestCase
from browse.domain import metadata
-from browse.services.document.author_affil import split_authors
+from arxiv.util.authors import split_authors
from browse.services.document.metadata import AbsMetaSession
from browse.services.search.search_authors import queries_for_authors, split_long_author_list
from tests import path_of_for_test
class TestAuthorLinkCreation(TestCase):
-
+
def test_basic(self):
out = queries_for_authors('')
self.assertIsInstance(out, list)
diff --git a/tests/test_tex2utf.py b/tests/test_tex2utf.py
deleted file mode 100644
index 5ec27cd24..000000000
--- a/tests/test_tex2utf.py
+++ /dev/null
@@ -1,204 +0,0 @@
-"""Tests for Tex to UTF8 parsing."""
-from unittest import TestCase
-
-from browse.services.util.tex2utf import tex2utf
-
-
-class TextTex2Utf(TestCase):
-
- def test_tex2utf(self):
- test_str = "abc def ghijk lmnop qrs tuv wxyz 1234567890 !@# $%^ &* () _-=+"
- utf_out = tex2utf(test_str)
- self.assertEqual(utf_out, test_str)
-
- utf_out = tex2utf(test_str, letters=False)
- self.assertEqual(utf_out, test_str)
-
- test_str = "\\'e"
- utf_out = tex2utf(test_str)
- self.assertEqual(utf_out, chr(0xe9))
- #\'e tex2UTF é \'E tex2UTF É \'{e} tex2UTF é {\'e} tex2UTF {é}
Submission history
From: {{ abs_meta.submitter.name|tex_to_utf if abs_meta.submitter.name != None }} [view email] +Submission history
From: {{ abs_meta.submitter.name|tex2utf if abs_meta.submitter.name != None }} [view email] {#- Extra message for proxy sites (i.e. Proxy line has username and id) -#} {#- TODO: revisit this. Logic for display follows classic but is strange; sometimes a proxy is just a person's name. -#} - {% if abs_meta.proxy != None and abs_meta.proxy|wordcount > 1 %} [via {{ abs_meta.proxy.split()[0]|tex_to_utf|upper }} proxy]{% endif %} + {% if abs_meta.proxy != None and abs_meta.proxy|wordcount > 1 %} [via {{ abs_meta.proxy.split()[0]|tex2utf|upper }} proxy]{% endif %}{%- for version_entry in abs_meta.version_history -%} {{ generate_version_entry(version_entry, abs_meta.version) }} @@ -133,14 +91,15 @@
Submission history
From: {{ abs_meta.submitter.name|tex_to_utf if abs_m-
+
- + new + (most recent mailing, with abstracts) + +
- recent + (last 5 mailings) + +
- current month's + {{archive_id}} listings +
- specific year/month: + + {# + +
+ {% for url, year in stats_by_year %} + {{year}} + {% endfor %} +
{{category.description}}
+
@@ -61,7 +55,7 @@
{% block header %}
{#- The ignore_me link is not meant to be visible to users; it is meant to catch robots/crawlers not respecting robots.txt. aria-hidden prevents screenreaders from being caught. -#}
- {% block header_h1 %}
diff --git a/browse/templates/cookies.html b/browse/templates/cookies.html
new file mode 100644
index 000000000..87c2bf2f1
--- /dev/null
+++ b/browse/templates/cookies.html
@@ -0,0 +1,81 @@
+{%- extends "base.html" -%}
+
+{% block title %}Your cookie configuration{% endblock %}
+
+{% block header_h1 %}arXiv.org
{% endblock header_h1%} + {% block header_h1 %}{{ config['BROWSE_SITE_LABEL'] or 'arXiv.org' }}
{% endblock header_h1%} {% block login_link %}{% endblock %}If you have a disability and are having trouble accessing information on this website or need materials in an alternate format, - contact web-accessibility@cornell.edu for assistance.
+ contact web-accessibility@cornell.edu for assistance.{{ config['BROWSE_SITE_LABEL'] }} > cookies
{% endblock header_h1 %} + +{%- block content %} +Your cookie configuration for arXiv
+ +You can use this form to set preferences that will apply to all arXiv.org sites for this +and future sessions when access from the same machine. Cookies are snippets of information +stored on your machine and returned to our site with each request you make (click +show additional debugging information below to see what is stored). +This facility will not work if you have disabled cookies or on browsers that do not +support cookies.
+ +For information about the full-text formats available, consult the +viewing help. +If the cookies are set successfully, then abstract and listing +pages will automatically link to the appropriate versions of each +paper. You may have to reload previously-visited abstract/listing +pages to update the format links (Shift+Reload in many browsers), because +the old versions of the pages will be cached by your browser.
++ + + +
+ +{% if debug %} +
Debugging information: dump of current cookie data
+| Name | Value | |
|---|---|---|
{{cooname}} |
+ {% else %}
+ {{cooname}} |
+ {% endif %}
+ {{ request.cookies[cooname] }} |
+
Only those cookies shown emphasized in the table above are controlled
+by this interface. arXiv uses other cookies such as browser and
+tapir_permanent to maintain session and permanent login settings.
+You can, if you want, clear all cookies using the controls in your browser.
(show additional debugging information)
+{% endif %} + +About cookies
+"HTTP Cookies" allow clients to retain certain state +information to be included in future requests to a particular +server. Cookies are now defined +by RFC 6265. It is +not necessary to use/permit cookies to read articles on arXiv.org.
+{% endblock content %} diff --git a/browse/templates/feedback_collector.html b/browse/templates/feedback_collector.html new file mode 100644 index 000000000..71143e7bd --- /dev/null +++ b/browse/templates/feedback_collector.html @@ -0,0 +1,2 @@ +Browse v0.2.1 released 2019-04-18 + diff --git a/browse/templates/feedback_collector_js.html b/browse/templates/feedback_collector_js.html index 885aabc79..b0c05510b 100644 --- a/browse/templates/feedback_collector_js.html +++ b/browse/templates/feedback_collector_js.html @@ -8,8 +8,8 @@ }); }, fieldValues: { - "components": ["15700"], // Browse component. - "versions": ["14132"], // Release browse-0.1 + "components": ["15700"], // Jira ID for browse component + "versions": ["14153"], // Jira ID for browse-0.2.1 release "customfield_11401": window.location.href } }; diff --git a/browse/templates/home/home.html b/browse/templates/home/home.html new file mode 100644 index 000000000..66961972d --- /dev/null +++ b/browse/templates/home/home.html @@ -0,0 +1,95 @@ +{%- extends "base.html" -%} +{% block title %}arXiv.org e-Print archive{% endblock %} +{% block head %} + {{ super() -}} +{% endblock head %} + +{% block login_link %}{% include 'login.html' %}{% endblock %} +{% block body_id %}id="front"{% endblock %} + +{%- block content %} +{#- TODO: display order in taxonomy? -#} +Open access to {% if document_count -%}{{ "{:,}".format(document_count) }}{%- endif %} + e-prints in the fields of physics, mathematics, computer science, quantitative biology, quantitative finance, statistics, electrical engineering and systems science, and economics. Submissions to arXiv should conform to Cornell University academic standards. arXiv is owned and operated by Cornell University, a private not-for-profit educational institution. arXiv is funded by Cornell University, the Simons Foundation and by the member institutions.
++{#- /multi sends to either search, catchup or form interface based on which button is hit. -#} + + + +
+{%- include "home/news.html" -%} +See cumulative "What's New" pages. +Read robots beware before attempting any automated download +
+{#- TODO: define display order in taxonomy? -#} +{{- group_section(('grp_physics','grp_math','grp_cs','grp_q-bio','grp_q-fin','grp_stat','grp_eess','grp_econ')) }} + ++{#- TODO: remove because of new footer? -#} +
About arXiv
+-
+
- General information and Scientific Advisory Board +
- Support and Governance Model and Member Advisory Board +
- Find, view, email alerts and RSS feeds +
- Submission and moderation details +
- Usage statistics and news +
- See also searchable help pages +
{{ groups[group_key].name }}
+-
+ {% for archive_key, archive_details in archives.items()|sort(attribute='1.name') if archives[archive_key].in_group == group_key %}
+ {%- set archive_search_url = url_for('search_archive', archive=archive_key) -%}
+ {%- set archive_url = url_for('browse.archive', archive=archive_key) if archive_key != 'cs' else 'https://arxiv.org/corr' -%}
+ {%- set archive_name = archives[archive_key].name if archive_key != 'cs' else 'Computing Research Repository' -%}
+
+ {#- TODO: url_for /list, /catchup -#}
+
-
+ {{ archive_name }} ({{ archive_key if archive_key != 'cs' else 'CoRR' }} new, recent, search{% if 0 %}, last month{% endif %})
+ {%- if group_key == 'grp_physics' and archive_key not in categories -%}
includes: + {%- elif archive_key == 'cs' -%}
includes (see detailed description): + {%- elif archive_key in ('eess', 'econ') -%}
includes (see detailed description): + {%- elif group_key != 'grp_physics' -%}
includes (see detailed description): + {%- endif -%} + {% for category_key, category_details in categories.items()|sort(attribute='1.name') if categories[category_key].in_archive == archive_key %} + {% if not (loop.first and loop.last and group_key == 'grp_physics') %}{{ categories[category_key].name }}{% if not loop.last %}; {% endif %}{% endif %} + {% endfor %} +
+ {% endfor %}
+
+5 Sept 2018: arXiv looks to the future with move to Cornell CIS
diff --git a/browse/templates/list/base.html b/browse/templates/list/base.html new file mode 100644 index 000000000..6f3673342 --- /dev/null +++ b/browse/templates/list/base.html @@ -0,0 +1,155 @@ +{%- extends "base.html" -%} + +{% block title %}{{list_ctx_name}} {{list_month_name}} {{list_year}}{% endblock %} + +{% block header_h1 %} +
+ {{ config['BROWSE_SITE_LABEL'] }} > + {{list_ctx_id}} +
+{% endblock %} + +{%- block content %} +
+
+{%- endblock %}
+
+
+{%- macro page_divs() -%}
+[ total of {{count}} entries
+ {{ pages() }}
+ ]
+
+[ showing {{shown}} entries per page:
+ {{ show_link('fewer',mf_fewer) }} |
+ {{ show_link('more', mf_more) }} |
+ {{ show_link('all', mf_all) }} ]
+
+{%- endmacro -%}
+
+
+{% macro item_list(items, heading) %}
+-
+ {% if heading %}
+
- + [{{listing_index}}] + + arXiv:{{ article.arxiv_identifier.ids }} + + {{ type_info( item ) }} + {{ dl_links( article ) }} + +
- + + + {% endfor %} +
{{heading}}
+ {% endif %} + + {% for item in items %} + {% set article = item['article'] %} + {% set ids = article.arxiv_identifier.ids %} + {% set listing_index = item['list_index'] %} +Authors and titles for {{pubmonth.strftime('%B %Y')}}
+{% endblock %} diff --git a/browse/templates/list/new.html b/browse/templates/list/new.html new file mode 100644 index 000000000..18b133599 --- /dev/null +++ b/browse/templates/list/new.html @@ -0,0 +1,38 @@ +{%- extends "list/base.html" -%} + +{# +New is quite different than the other list pages. +It has an index of types of items. +It needs to display the submitted dates and announced dates. +It needs to do headings for the new/cross/rep sections. +#} + +{% block list_index %} +{{ listdate() }} +{{ type_index() }} +{% endblock %} + +{% block items %} + {% for sec in sub_sections_for_types %} + {{ item_list(sec['items'], sec['heading']) }} + {% endfor %} +{% endblock %} + +{% macro type_index() %} + {% if index_for_types %} +-
+ {% for ityp in index_for_types %}
+
- {{ityp[0]}} + {% endfor %} +
Submissions received from
+ {{ submitted[0].strftime('%A, %-d %B %Y') }} to {{ submitted[1].strftime('%A, %-d %B %Y') }},
+ announced {{ announced.strftime('%A, %-d %B %Y') }}
+
+ {% endif %}
+{% endmacro %}
diff --git a/browse/templates/list/recent.html b/browse/templates/list/recent.html
new file mode 100644
index 000000000..7e9c578ea
--- /dev/null
+++ b/browse/templates/list/recent.html
@@ -0,0 +1,24 @@
+{%- extends "list/base.html" -%}
+
+{% block list_index %}
+Authors and titles for recent submissions
+ +-
+ {% for pdate in pubdates %}
+ {% set skipfordate = pdate[1] %}
+
- + + {{ pdate[0].strftime('%A, %-d %B %Y') }} + + + {% endfor %} +
Submissions announced {{ announced.strftime('%A, %-d %B %Y') }}
+
+ {% endif %}
+{% endmacro %}
diff --git a/browse/templates/list/year.html b/browse/templates/list/year.html
new file mode 100644
index 000000000..8391a4dbe
--- /dev/null
+++ b/browse/templates/list/year.html
@@ -0,0 +1,5 @@
+{%- extends "list/base.html" -%}
+
+{% block list_index %}
+Authors and titles for {{pubmonth.strftime('%Y')}}
+{% endblock %} diff --git a/browse/templates/login.html b/browse/templates/login.html new file mode 100644 index 000000000..a3cbf2e76 --- /dev/null +++ b/browse/templates/login.html @@ -0,0 +1,7 @@ +
+{% if request.auth.user is defined %}
+ Logged in as {{ request.auth.user.username }} | My Account | Logout
+{% else %}
+ Login
+{% endif %}
+
diff --git a/browse/templates/stats/base.html b/browse/templates/stats/base.html
new file mode 100644
index 000000000..f36e88a98
--- /dev/null
+++ b/browse/templates/stats/base.html
@@ -0,0 +1,14 @@
+{%- extends "base.html" -%}
+{# Do not show login status on stats pages for now, per classic #}
+
+{% block head %}
+ {{ super() -}}
+ {# TODO: upgrade d3 to v5.x? #}
+
+{% endblock head %}
+
+{% block header_h1 %}{{ config['BROWSE_SITE_LABEL'] }} > stats > server usage
{% endblock %} + +{%- block content %} +See also other arXiv usage statistics.
+{% endblock content %} diff --git a/browse/templates/stats/monthly_downloads.html b/browse/templates/stats/monthly_downloads.html new file mode 100644 index 000000000..222fd5ec9 --- /dev/null +++ b/browse/templates/stats/monthly_downloads.html @@ -0,0 +1,21 @@ +{%- extends "stats/base.html" -%} + +{% block title %}Monthly Download Rates{% endblock %} +{% block header_h1 %}{{ config['BROWSE_SITE_LABEL'] }} > stats > monthly downloads
{% endblock %} + +{% block content %} +arXiv Monthly Download Rates [CSV]
+ +{% include "stats/monthly_downloads_js.html" %} + +Blue: Number of downloads per month.
Click and drag in the small graph below to pan or zoom.
Total number of downloads {% if most_recent_dt %}through {{ most_recent_dt.strftime('%B %Y') }}{% endif %} = {{ "{:,}".format(total_downloads) }}
++ Caveats: While we have attempted to extract download data representing unique full-text downloads by real users, there are many factors which affect accuracy. These factors include: 1) the data is from the main arXiv site and the + arXiv mirrors, though some mirror data is incomplete; 2) only web downloads are included (and not FTP or email "downloads" that were formerly supported); 3) we have counted downloads according to the COUNTER algorithm which excludes + rapid repeat downloads; 4) we have attempted to identify and remove robot or automated downloads from the count (false positives lead to undercounting, failing to identify robots leads to overcounting); 5) data prior to 2009 has not been cleaned + with as much care as later data, it shows trends nonetheless. +
+ +{{ super() }} +{% endblock content %} diff --git a/browse/templates/stats/monthly_downloads_js.html b/browse/templates/stats/monthly_downloads_js.html new file mode 100644 index 000000000..84833c98d --- /dev/null +++ b/browse/templates/stats/monthly_downloads_js.html @@ -0,0 +1,160 @@ + + diff --git a/browse/templates/stats/monthly_submissions.html b/browse/templates/stats/monthly_submissions.html new file mode 100644 index 000000000..216f26fb8 --- /dev/null +++ b/browse/templates/stats/monthly_submissions.html @@ -0,0 +1,16 @@ +{%- extends "stats/base.html" -%} + +{% block title %}Monthly Submissions{% endblock %} +{% block header_h1 %}{{ config['BROWSE_SITE_LABEL'] }} > stats > monthly submission rates
{% endblock %} + +{% block content %} +arXiv Monthly Submission Rates [CSV]
+{% include "stats/monthly_submissions_js.html" %} +Blue: Number of new submissions received during each month since {{ arxiv_start_dt.strftime('%B %Y') }}.
+ Hover over the graph to see the exact count for a given month.
Total number of submissions shown in graph as of {{ current_dt.strftime('%B %-d, %Y') }} (after {{ "%.1f"|format(arxiv_age_years|float) }} years) = {{ "{:,}".format(num_submissions) }}
++ The total number of submissions excludes {{ "{:,}".format(num_migrated) }} articles that were migrated to arXiv rather than being submitted directly, and includes {{ "{:,}".format(num_deleted) }} articles that have been deleted. The total number of articles available is {{ "{:,}".format(num_submissions_adjusted) }}. +
+{{ super() }} +{% endblock content %} diff --git a/browse/templates/stats/monthly_submissions_js.html b/browse/templates/stats/monthly_submissions_js.html new file mode 100644 index 000000000..b8f9bd946 --- /dev/null +++ b/browse/templates/stats/monthly_submissions_js.html @@ -0,0 +1,199 @@ + + + diff --git a/browse/templates/stats/today.html b/browse/templates/stats/today.html new file mode 100644 index 000000000..2309d99a0 --- /dev/null +++ b/browse/templates/stats/today.html @@ -0,0 +1,16 @@ +{%- extends "stats/base.html" -%} + +{% block title %}hourly usage{% endblock %} + +{% block content %} + +arXiv Web Server Usage from {{ requested_dt.strftime('%A, %B %-d, %Y') }} ({{ config['BROWSE_SITE_LABEL'] }} site only) [CSV]
+{% if normal_count > 0 %} +{% include "stats/today_js.html" %} +Total number of connections = {{ "{:,}".format(normal_count) }} (+{{ "{:,}".format(admin_count) }} local & administrative connections) +{% else %} +
No data currently available.
+{% endif %}
+
Current local time is {{ current_dt.strftime('%A, %B, %-d, %Y %H:%M%:%S') }} US/Eastern
Article 'arXiv:{{ arxiv_id }}' not found
+The identifier you have specified (arXiv:{{ arxiv_id }}) may be invalid. Please inform help@arxiv.org if you believe that the identifier should correspond to a valid paper in arXiv.
+ {% elif arxiv_id %} +Article identifier '{{ arxiv_id }}' not recognized
+ {% else %} +No article identifier specified
+ {% endif %} +We were unable to extract an arXiv article ID from the URL specified. You may:
+{% else %} +Trackback not found
+Badly formed redirect request. If the link you followed was on arXiv, please report this error to us. If it was elsewhere, we suggest you report it to the maintainer of that site.
+{% endif %} +-
+
- View recent trackbacks +
- View the trackbacks for a particular article at
https://arxiv.org{{ url_for('.tb', arxiv_id='') }}{arXiv_id}
+
+
+
+{% endblock content %}
diff --git a/browse/templates/tb/macros.html b/browse/templates/tb/macros.html
new file mode 100644
index 000000000..4b93e4531
--- /dev/null
+++ b/browse/templates/tb/macros.html
@@ -0,0 +1,7 @@
+{%- macro generate_trackback_link(trackback, include_posted_date=True) -%}
+
+ {{- trackback.title|entity_to_utf|truncate(150) }} [
+ {%- if trackback.blog_name %}{{ trackback.blog_name|trim|entity_to_utf|truncate(50) -}} @ {% endif -%}
+ {%- if trackback.has_valid_url %}{{ trackback.display_url|truncate(30,False) }}{% else %}INVALID-URL{% endif -%}]
+ {%- if include_posted_date %} [trackback posted {{ trackback.posted_datetime.strftime('%a, %-d %b %Y %H:%M:%S %Z') }}]{% endif %}
+{%- endmacro -%}
diff --git a/browse/templates/tb/recent.html b/browse/templates/tb/recent.html
new file mode 100644
index 000000000..efffeb707
--- /dev/null
+++ b/browse/templates/tb/recent.html
@@ -0,0 +1,42 @@
+{%- extends "tb/base.html" -%}
+{%- from 'tb/macros.html' import generate_trackback_link -%}
+{%- import 'base/macros.html' as base_macros -%}
+
+{% block title %}Recent Trackbacks{% endblock %}
+{% block header_h1 %}About trackbacks
+By sending a trackback, you can notify + arXiv.org that you have created a web page that references a paper. Popular + blogging software supports trackback: you can send us a trackback about this + paper by giving your software the following trackback URL:
+https://arxiv.org/trackback/{arXiv_id}+
Some blogging software supports trackback autodiscovery -- in this + case, your software will automatically send a trackback as soon as your + create a link to our abstract page. See our + trackback help page for more information.
+{{ config['BROWSE_SITE_LABEL'] }} > recent trackbacks
{% endblock %} + +{%- block content %} +Recent Trackbacks
+Trackbacks indicate external web sites that link to articles in arXiv.org. Trackbacks may not reflect the opinion of arXiv.org or of that article's authors.
+ + {% if recent_trackback_pings -%} + {%- set ns = namespace(cur_tb_ymd='', cur_tb_url='') -%} + {%- for recent_tb in recent_trackback_pings -%} + {%- set this_tb_ymd = recent_tb[0].posted_datetime.strftime('%B %-d, %Y') -%} + {%- set this_tb_url = recent_tb[0].url -%} + {%- if this_tb_ymd != ns.cur_tb_ymd -%} + {%- set ns.cur_tb_ymd = this_tb_ymd -%} +{{ this_tb_ymd }}
+ {%- endif -%} + {%- if this_tb_url != ns.cur_tb_url -%} + {%- set ns.cur_tb_url = this_tb_url -%} +{{- generate_trackback_link(recent_tb[0], include_posted_date=False) }} links the following articles:
+-
+ {%- for article in article_map[this_tb_url] -%}
+
- {{ article[1]|tex2utf }} [{{ article[0] }}] + {%- endfor -%} +
{{ config['BROWSE_SITE_LABEL'] }} > article trackbacks
{% endblock %} +{# Disable login on this page for now, per classic #} +{% block login_link %}{% endblock %} + +{%- block content %} + {% if trackback_pings %} +Trackbacks for {{ arxiv_identifier.id }}
+ {% for tb in trackback_pings %} +{{- generate_trackback_link(tb) -}}
+ {% endfor %} ++ {#- abstract field is deliberately suppressed in call to abs macro. -#} + {%- if abs_meta -%} + {{ base_macros.abs( + abs_meta.arxiv_identifier.id, + abs_meta.title, + display_authors_with_links(abs_meta, author_links), + '', + abs_meta.get_datetime_of_version(abs_meta.version), + abs_meta.primary_category.id, + comments = abs_meta.comments, + msc_class = abs_meta.msc_class, + acm_class = abs_meta.acm_class, + journal_ref = abs_meta.journal_ref, + doi = abs_meta.doi, + report_num = abs_meta.report_num, + version = abs_meta.version, + submission_history = abs_meta.version_history, + secondary_categories = abs_meta.get_secondaries()) }} ++ {%- endif -%} + {% else %} +
There are no trackback pings recorded for {{ arxiv_identifier.id }}.
+ {% endif %} + {{ super() }} +{% endblock content %} diff --git a/browse/templates/user_banner.html b/browse/templates/user_banner.html index 19d33ec10..8ec524811 100644 --- a/browse/templates/user_banner.html +++ b/browse/templates/user_banner.html @@ -1,4 +1,19 @@ -
-
+
diff --git a/browse/templates/year.html b/browse/templates/year.html
new file mode 100644
index 000000000..d3c205c10
--- /dev/null
+++ b/browse/templates/year.html
@@ -0,0 +1,54 @@
+{% extends "base.html" %}
+
+{% block title %}{{archive.name}}{% endblock %}
+
+
+{% block header_h1 %}
+Article statistics by year:
+ {% for url, year in stats_by_year %} + {% if url %} + {{year}} + {% else %} + {{year}} + {% endif %} + {% endfor %} +
+
+
+
+{% endblock %}
diff --git a/browse/util/__init__.py b/browse/util/__init__.py
deleted file mode 100644
index a7a09d0fa..000000000
--- a/browse/util/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-"""Utility code for browse."""
diff --git a/browse/util/clickthrough.py b/browse/util/clickthrough.py
deleted file mode 100644
index 2e8b849cb..000000000
--- a/browse/util/clickthrough.py
+++ /dev/null
@@ -1,30 +0,0 @@
-"""
-Functions to create hashes for clickthrough links.
-
-These are used to log to the web acccess logs when the DOI and
-bookmarking links are clicked.
-
-The hash is used to prevent malicious use of the click through
-controller to create links that look like they are on arXiv but get
-redirected someplace undesirable.
-"""
-
-from typing import Callable
-import hashlib
-
-
-def create_hash(secret: str, url: str) -> str:
- """Create a hash of the secret and url."""
- s = f'{secret}{url}'
- return str(hashlib.md5(s.encode()).hexdigest()[0:8])
-
-
-def is_hash_valid(secret: str, url: str, ct_hash: str) -> bool:
- """Check that ct_hash was generated by create_hash for secret and url."""
- return ct_hash == create_hash(secret, url)
-
-
-def create_ct_url(secret: str, url_for: Callable[..., str], url: str) -> str:
- """Create a URL to the clickthrough service with a valid hash."""
- url_for('browse.clickthrough', url=url, v=create_hash(secret, url))
- return url_for('browse.clickthrough', url=url, v=create_hash(secret, url))
diff --git a/browse/util/id_patterns.py b/browse/util/id_patterns.py
deleted file mode 100644
index 325242fa7..000000000
--- a/browse/util/id_patterns.py
+++ /dev/null
@@ -1,329 +0,0 @@
-r"""Patterns and functions to detect arXiv ids and Urls in text.
-
-Functions to detech arXiv ids, URLs and DOI in text.
-Functions to transform them to tags.
-
-These were originally jinja filters but became a little too big
-for that so they were split out and made more general so they didn't
-rely on the Flask context.
-
-These all use expect input of Markup or non-markup text and return
-Markup objects. This is because the that get added need to avoid
-double escaping.
-
-There are several classes of patterns we want to match but there is
-some overlap in these patterns. To avoid looking for and parsing HTML in each
-jinja filter, detecting these patterns is combined.
-
-So far we are looking for:
-DOIs DOI: 10.1145/0001234.1234567
-arXiv IDS: 1234.12345 1234.12345v1 hep-ph1307.1843
-HTTP URLs: http://something.org/myPaper/1234.12345
-FTP URLs: ftp://example.com/files/1234.12345
-
-Just matching for arXiv ids with \d{4}\.\d{4,5} will match several of
-these. To deal with this we are priortizing the matches and
-interupting once one is found.
-
-We should probably match DOIs first because they are the source of a
-lot of false positives for arxiv matches.
-"""
-from typing import Optional, List, Pattern, Match, Tuple, Callable
-import re
-from dataclasses import dataclass
-
-from urllib.parse import quote
-from jinja2 import Markup, escape
-
-from arxiv import taxonomy
-
-
-@dataclass
-class Matchable:
- """Class for paterns."""
-
- examples: List[str]
- pattern: Pattern
-
-
-def _identity(x: str)->str:
- """Identity funciton for default in some places."""
- return x
-
-
-doi_patterns = [
- Matchable(['10.1145/0001234.1234567'],
- re.compile(r'(?P10.\d{4,9}/[-._;()/:A-Z0-9]+)', re.I))
-]
-"""List of Matchable for DOIs in text.
-
-We should probably match DOIs first because they are the source of a
-lot of false positives for arxiv matches.
-
-Only using the most general express from
-https://www.crossref.org/blog/dois-and-matching-regular-expressions/
-"""
-
-_archive = '|'.join([re.escape(key) for key in taxonomy.definitions.ARCHIVES.keys()])
-"""string for use in Regex for all arXiv archives"""
-
-_category = '|'.join([re.escape(key) for key in taxonomy.definitions.CATEGORIES.keys()])
-
-_arxiv_id_prefix = r'(?Par[xX]iv:)?'
-"""Attempt to catch the arxiv prefix in front of arxiv ids so it can be
-included in the tag anchor. ARXIVNG-1284"""
-
-basic_arxiv_id_patterns = [
- Matchable(['math/0501233', 'hep-ph/0611734', 'gr-qc/0112123'],
- re.compile(_arxiv_id_prefix + r'(?P(%s)\/\d{2}[01]\d{4}(v\d*)?)'
- % _archive, re.I)),
- Matchable(['1609.05068', '1207.1234v1', '1207.1234', '1807.12345',
- '1807.12345v1', '1807.12345v12'],
- re.compile(r'(?\d{4}\.\d{4,5}(v\d*)?)',
- re.I)),
- Matchable(['math.GR/0601136v3', 'math.GR/0601136'],
- re.compile(_arxiv_id_prefix + r'(?P(%s)\/\d{2}[01]\d{4}(v\d*)?)'
- % _category, re.I))
-]
-
-OKCHARS = r'([a-z0-9,_.\-+~:]|%[a-f0-9]*)'
-"""Chacters that are acceptable during PATH, QUERY and ANCHOR parts"""
-
-HOST_NAME = r'(?:[a-z0-9][a-z0-9\-.:]+[a-z0-9])'
-"""Regex used to match host names in arXiv urlize.
-
-This is not a perfect regex for a host name, It accepts only a sub-set
-of hostnames to meet the needs of arxiv.
-
-HOST_NAME must end with a simplified character to avoid capturing a
-period.
-"""
-
-PATH = rf'(?P(/{OKCHARS}*)+)?'
-"""Regex for path part of URLs for use in urlize"""
-
-QUERY = rf'(?P\?(&?({OKCHARS}*(={OKCHARS}*)?))*)?'
-"""Regex for query part of URLs for use in urlize"""
-
-ANCHOR = rf'(?P#({OKCHARS}|/)*)?'
-"""Regex for anchor part of URLs for use in urlize"""
-
-URLINTEXT_PAT = re.compile(r'(?P(?:https?://)'
- f'{HOST_NAME}{PATH}{QUERY}{ANCHOR})',
- re.I)
-"""Regex to match URLs in text."""
-
-FTP_PAT = re.compile(rf'(?P(?:ftp://)({OKCHARS}|(@))*{PATH})', re.I)
-"""Regex to match FTP URLs in text."""
-
-basic_url_patterns = [
- Matchable(['http://something.com/bla'], URLINTEXT_PAT),
- Matchable(['ftp://something.com/bla'], FTP_PAT)
-]
-"""List of Matchable to use when finding URLs in text"""
-
-bad_arxiv_id_patterns = [
- re.compile('vixra', re.I), # don't need to link to vixra
-]
-"""List of Regex patterns that will cause matching to be skipped for
-the token."""
-
-dois_ids_and_urls = basic_url_patterns + doi_patterns + basic_arxiv_id_patterns
-"""List of Matchable to use when finding DOIs, arXiv IDs, and URLs.
-
-URLs are first because some URLs contain DOIs or arXiv IDS.
-
-DOI are before arXiv ids because many DOIs are falsely matched by the
-arxiv_id patterns.
-"""
-
-
-_bad_endings = ['.', ',', ':', ';', '&', '(', '[', '{']
-"""These should not appear at the end of URLs because they are likely
-part of the surrounding text"""
-
-
-def _find_match(patterns: List[Matchable], token: str) \
- -> Optional[Tuple[Match, Matchable]]:
- """Find first in patterns that is found in txt."""
- for chgMtch in patterns:
- if chgMtch.pattern.flags:
- fnd = re.search(chgMtch.pattern, token)
- else:
- fnd = re.search(chgMtch.pattern, token, re.I)
- if fnd is not None:
- return (fnd, chgMtch)
- return None
-
-
-def _transform_token(patterns: List[Matchable],
- bad_patterns: List[Pattern],
- id_to_url: Callable[[str], str],
- doi_to_url: Callable[[str], str],
- url_to_url: Callable[[str], str],
- token: str) -> str:
- """
- Transform a token from text to one of the Matchables.
-
- This only transforms against the first of Matchable matched.
- Matching on this token will be skipped if any of the bad_patterns
- match the token (that is re.search).
- """
- id_to_url = id_to_url or (lambda x: x)
- doi_to_url = doi_to_url or (lambda x: x)
- url_to_url = url_to_url or (lambda x: x)
-
- for pattern in bad_patterns:
- if re.search(pattern, token):
- return token
-
- mtch = _find_match(patterns, token)
- if mtch is None:
- return token
-
- (match, _) = mtch
- keys = match.groupdict().keys()
- if 'arxiv_id' in keys:
- (front, back) = _arxiv_id_sub(match, id_to_url)
- elif 'doi' in keys:
- (front, back) = _doi_sub(match, doi_to_url)
- elif 'url' in keys:
- (front, back) = _url_sub(match, url_to_url)
- else:
- # unclear how to substitute in for this match
- return token
-
- if back:
- t_back = _transform_token(patterns, bad_patterns,
- id_to_url, doi_to_url, url_to_url, back)
- return front + Markup(t_back)
- else:
- return front
-
-
-def _arxiv_id_sub(match: Match, id_to_url: Callable[[str], str]) \
- -> Tuple[Markup, str]:
- """Return match.string transformed for a arxiv id match."""
- aid = match.group('arxiv_id')
- prefix = 'arXiv:' if match.group('arxiv_prefix') else ''
-
- if aid[-1] in _bad_endings:
- arxiv_url = id_to_url(aid)[:-1]
- anchor = aid[:-1]
- back = aid[-1] + match.string[match.end():]
- else:
- arxiv_url = id_to_url(aid)
- anchor = prefix + aid
- back = match.string[match.end():]
-
- front = match.string[0:match.start()]
- return (Markup(f'{front}{anchor}'), back)
-
-
-def _doi_sub(match: Match, doi_to_url: Callable[[str], str]) \
- ->Tuple[Markup, str]:
- """Return match.string transformed for a DOI match."""
- doi = match.group('doi')
- if(doi[-1] in _bad_endings):
- back = match.string[match.end():] + doi[-1]
- doi = doi[:-1]
- else:
- back = match.string[match.end():]
-
- quoted_doi = quote(doi, safe='/')
- doi_url = f'https://dx.doi.org/{quoted_doi}'
- doi_url = doi_to_url(doi_url)
-
- anchor = escape(doi)
- front = match.string[0:match.start()]
- return (Markup(f'{front}{anchor}'), back)
-
-
-def _url_sub(match: Match, url_to_url: Callable[[str], str]) \
- ->Tuple[Markup, str]:
- """Return match.string transformed for a URL match."""
- url = match.group('url')
- if url.startswith('https'):
- anchor = 'this https URL'
- elif url.startswith('http'):
- anchor = 'this http URL'
- elif url.startswith('ftp'):
- anchor = 'this ftp URL'
- else:
- anchor = 'this URL'
-
- front = match.string[0:match.start()]
- if url[-1] in _bad_endings:
- back = url[-1] + match.string[match.end():]
- url = url[:-1]
- else:
- back = match.string[match.end():]
-
- url = url_to_url(url)
- return (Markup(f'{front}{anchor}'), back)
-
-
-_word_split_re = re.compile(r'(\s+)')
-"""Regex to split to tokens during _to_tags.
-
-Capturing group causes the splitting spaces to be included
-in the returned list.
-"""
-
-
-def _to_tags(patterns: List[Matchable],
- bad_patterns: List[Pattern],
- id_to_url: Callable[[str], str],
- doi_to_url: Callable[[str], str],
- url_to_url: Callable[[str], str],
- text: str)-> str:
- """Split text to tokens, do _transform_token for each, return results."""
- def transform_token(tkn: str)-> str:
- return _transform_token(patterns, bad_patterns,
- id_to_url, doi_to_url, url_to_url,
- tkn)
-
- if not hasattr(text, '__html__'):
- text = Markup(escape(text))
-
- words = _word_split_re.split(text)
- for i, token in enumerate(words):
- token_2 = transform_token(token)
- if token_2 != token:
- words[i] = token_2
- result = u''.join(words)
- return Markup(result)
-
-
-def do_id_to_tags(id_to_url: Callable[[str], str],
- text: str)-> str:
- """Transform arxiv ids in text to tags."""
- return _to_tags(basic_arxiv_id_patterns,
- bad_arxiv_id_patterns,
- id_to_url, _identity, _identity,
- text)
-
-
-def do_dois_id_urls_to_tags(id_to_url: Callable[[str], str],
- doi_to_url: Callable[[str], str],
- text: str)-> str:
- """Transform DOIs, arxiv ids and URLs in text to tags."""
- return _to_tags(dois_ids_and_urls,
- bad_arxiv_id_patterns,
- id_to_url, doi_to_url, _identity,
- text)
-
-
-def do_dois_to_tags(doi_to_url: Callable[[str], str], text: str)->str:
- """Transform DOIs in text to tags."""
- return _to_tags(doi_patterns, [], _identity, doi_to_url, _identity, text)
-
-
-def do_dois_arxiv_ids_to_tags(id_to_url: Callable[[str], str],
- doi_to_url: Callable[[str], str],
- text: str)->str:
- """Transform DOIs and arXiv IDs to tags."""
- return _to_tags(doi_patterns + basic_arxiv_id_patterns,
- bad_arxiv_id_patterns,
- id_to_url, doi_to_url, _identity,
- text)
diff --git a/docs/Makefile b/docs/Makefile
new file mode 100644
index 000000000..69fe55ecf
--- /dev/null
+++ b/docs/Makefile
@@ -0,0 +1,19 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS =
+SPHINXBUILD = sphinx-build
+SOURCEDIR = source
+BUILDDIR = build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+ @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+ @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
\ No newline at end of file
diff --git a/docs/source/api/browse.config.rst b/docs/source/api/browse.config.rst
new file mode 100644
index 000000000..db15bb759
--- /dev/null
+++ b/docs/source/api/browse.config.rst
@@ -0,0 +1,7 @@
+browse.config module
+========================================
+
+.. automodule:: browse.config
+ :members:
+ :undoc-members:
+ :show-inheritance:
diff --git a/docs/source/api/browse.controllers.abs_page.rst b/docs/source/api/browse.controllers.abs_page.rst
new file mode 100644
index 000000000..060b7327f
--- /dev/null
+++ b/docs/source/api/browse.controllers.abs_page.rst
@@ -0,0 +1,7 @@
+browse.controllers.abs_page module
+========================================
+
+.. automodule:: browse.controllers.abs_page
+ :members:
+ :undoc-members:
+ :show-inheritance:
diff --git a/docs/source/api/browse.controllers.home_page.rst b/docs/source/api/browse.controllers.home_page.rst
new file mode 100644
index 000000000..7d050838e
--- /dev/null
+++ b/docs/source/api/browse.controllers.home_page.rst
@@ -0,0 +1,7 @@
+browse.controllers.home_page module
+========================================
+
+.. automodule:: browse.controllers.home_page
+ :members:
+ :undoc-members:
+ :show-inheritance:
diff --git a/docs/source/api/browse.controllers.list_page.rst b/docs/source/api/browse.controllers.list_page.rst
new file mode 100644
index 000000000..d337a6bdd
--- /dev/null
+++ b/docs/source/api/browse.controllers.list_page.rst
@@ -0,0 +1,7 @@
+browse.controllers.list_page module
+========================================
+
+.. automodule:: browse.controllers.list_page
+ :members:
+ :undoc-members:
+ :show-inheritance:
diff --git a/docs/source/api/browse.controllers.prevnext.rst b/docs/source/api/browse.controllers.prevnext.rst
new file mode 100644
index 000000000..3f6cefa63
--- /dev/null
+++ b/docs/source/api/browse.controllers.prevnext.rst
@@ -0,0 +1,7 @@
+browse.controllers.prevnext module
+========================================
+
+.. automodule:: browse.controllers.prevnext
+ :members:
+ :undoc-members:
+ :show-inheritance:
diff --git a/docs/source/api/browse.controllers.rst b/docs/source/api/browse.controllers.rst
new file mode 100644
index 000000000..0eff93e9b
--- /dev/null
+++ b/docs/source/api/browse.controllers.rst
@@ -0,0 +1,17 @@
+browse.controllers package
+==========================
+
+.. automodule:: browse.controllers
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
+Subpackages
+-----------
+
+.. toctree::
+
+ browse.controllers.abs_page
+ browse.controllers.home_page
+ browse.controllers.list_page
+ browse.controllers.prevnext
\ No newline at end of file
diff --git a/docs/source/api/browse.domain.category.rst b/docs/source/api/browse.domain.category.rst
new file mode 100644
index 000000000..58a599871
--- /dev/null
+++ b/docs/source/api/browse.domain.category.rst
@@ -0,0 +1,7 @@
+browse.domain.category module
+=============================
+
+.. automodule:: browse.domain.category
+ :members:
+ :undoc-members:
+ :show-inheritance:
diff --git a/docs/source/api/browse.domain.identifier.rst b/docs/source/api/browse.domain.identifier.rst
new file mode 100644
index 000000000..644c40ea1
--- /dev/null
+++ b/docs/source/api/browse.domain.identifier.rst
@@ -0,0 +1,7 @@
+browse.domain.identifier module
+===============================
+
+.. automodule:: browse.domain.identifier
+ :members:
+ :undoc-members:
+ :show-inheritance:
diff --git a/docs/source/api/browse.domain.institution.rst b/docs/source/api/browse.domain.institution.rst
new file mode 100644
index 000000000..434608d81
--- /dev/null
+++ b/docs/source/api/browse.domain.institution.rst
@@ -0,0 +1,7 @@
+browse.domain.institution module
+================================
+
+.. automodule:: browse.domain.institution
+ :members:
+ :undoc-members:
+ :show-inheritance:
diff --git a/docs/source/api/browse.domain.license.rst b/docs/source/api/browse.domain.license.rst
new file mode 100644
index 000000000..d6b7422c8
--- /dev/null
+++ b/docs/source/api/browse.domain.license.rst
@@ -0,0 +1,7 @@
+browse.domain.license module
+=============================
+
+.. automodule:: browse.domain.license
+ :members:
+ :undoc-members:
+ :show-inheritance:
diff --git a/docs/source/api/browse.domain.listing.rst b/docs/source/api/browse.domain.listing.rst
new file mode 100644
index 000000000..37a94f172
--- /dev/null
+++ b/docs/source/api/browse.domain.listing.rst
@@ -0,0 +1,7 @@
+browse.domain.listing module
+=============================
+
+.. automodule:: browse.domain.listing
+ :members:
+ :undoc-members:
+ :show-inheritance:
diff --git a/docs/source/api/browse.domain.metadata.rst b/docs/source/api/browse.domain.metadata.rst
new file mode 100644
index 000000000..f8cf62a04
--- /dev/null
+++ b/docs/source/api/browse.domain.metadata.rst
@@ -0,0 +1,7 @@
+browse.domain.metadata module
+===============================
+
+.. automodule:: browse.domain.metadata
+ :members:
+ :undoc-members:
+ :show-inheritance:
diff --git a/docs/source/api/browse.domain.rst b/docs/source/api/browse.domain.rst
new file mode 100644
index 000000000..1add291d4
--- /dev/null
+++ b/docs/source/api/browse.domain.rst
@@ -0,0 +1,19 @@
+browse.domain package
+=====================
+
+.. automodule:: browse.domain
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
+Submodules
+----------
+
+.. toctree::
+
+ browse.domain.category
+ browse.domain.identifier
+ browse.domain.institution
+ browse.domain.license
+ browse.domain.listing
+ browse.domain.metadata
diff --git a/docs/source/api/browse.exceptions.rst b/docs/source/api/browse.exceptions.rst
new file mode 100644
index 000000000..b228e3e2d
--- /dev/null
+++ b/docs/source/api/browse.exceptions.rst
@@ -0,0 +1,7 @@
+browse.exceptions module
+========================================
+
+.. automodule:: browse.exceptions
+ :members:
+ :undoc-members:
+ :show-inheritance:
diff --git a/docs/source/api/browse.factory.rst b/docs/source/api/browse.factory.rst
new file mode 100644
index 000000000..3de36913e
--- /dev/null
+++ b/docs/source/api/browse.factory.rst
@@ -0,0 +1,7 @@
+browse.factory module
+========================================
+
+.. automodule:: browse.factory
+ :members:
+ :undoc-members:
+ :show-inheritance:
diff --git a/docs/source/api/browse.filters.rst b/docs/source/api/browse.filters.rst
new file mode 100644
index 000000000..a45502856
--- /dev/null
+++ b/docs/source/api/browse.filters.rst
@@ -0,0 +1,7 @@
+browse.filters module
+========================================
+
+.. automodule:: browse.filters
+ :members:
+ :undoc-members:
+ :show-inheritance:
diff --git a/docs/source/api/browse.routes.rst b/docs/source/api/browse.routes.rst
new file mode 100644
index 000000000..c68fe08c3
--- /dev/null
+++ b/docs/source/api/browse.routes.rst
@@ -0,0 +1,14 @@
+browse.routes package
+=====================
+
+.. automodule:: browse.routes
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
+Submodules
+----------
+
+.. toctree::
+
+ browse.routes.ui
diff --git a/docs/source/api/browse.routes.ui.rst b/docs/source/api/browse.routes.ui.rst
new file mode 100644
index 000000000..2f1c7c58f
--- /dev/null
+++ b/docs/source/api/browse.routes.ui.rst
@@ -0,0 +1,7 @@
+browse.routes.ui module
+=======================
+
+.. automodule:: browse.routes.ui
+ :members:
+ :undoc-members:
+ :show-inheritance:
diff --git a/docs/source/api/browse.rst b/docs/source/api/browse.rst
new file mode 100644
index 000000000..076cff262
--- /dev/null
+++ b/docs/source/api/browse.rst
@@ -0,0 +1,28 @@
+browse package
+==============
+
+.. automodule:: browse
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
+Subpackages
+-----------
+
+.. toctree::
+
+ browse.controllers
+ browse.domain
+ browse.routes
+ browse.services
+ browse.util
+
+Submodules
+----------
+
+.. toctree::
+
+ browse.config
+ browse.exceptions
+ browse.factory
+ browse.filters
diff --git a/docs/source/api/browse.services.database.models.rst b/docs/source/api/browse.services.database.models.rst
new file mode 100644
index 000000000..fc7530bc1
--- /dev/null
+++ b/docs/source/api/browse.services.database.models.rst
@@ -0,0 +1,7 @@
+browse.services.database.models module
+======================================
+
+.. automodule:: browse.services.database.models
+ :members:
+ :undoc-members:
+ :show-inheritance:
diff --git a/docs/source/api/browse.services.database.rst b/docs/source/api/browse.services.database.rst
new file mode 100644
index 000000000..d2bef259f
--- /dev/null
+++ b/docs/source/api/browse.services.database.rst
@@ -0,0 +1,14 @@
+browse.services.database package
+================================
+
+.. automodule:: browse.services.database
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
+Subpackages
+-----------
+
+.. toctree::
+
+ browse.services.database.models
diff --git a/docs/source/api/browse.services.document.author_affil.rst b/docs/source/api/browse.services.document.author_affil.rst
new file mode 100644
index 000000000..1322b6ccc
--- /dev/null
+++ b/docs/source/api/browse.services.document.author_affil.rst
@@ -0,0 +1,7 @@
+browse.services.document.author_affil module
+============================================
+
+.. automodule:: browse.services.document.author_affil
+ :members:
+ :undoc-members:
+ :show-inheritance:
diff --git a/docs/source/api/browse.services.document.cache.rst b/docs/source/api/browse.services.document.cache.rst
new file mode 100644
index 000000000..684e37e14
--- /dev/null
+++ b/docs/source/api/browse.services.document.cache.rst
@@ -0,0 +1,7 @@
+browse.services.document.cache module
+=====================================
+
+.. automodule:: browse.services.document.cache
+ :members:
+ :undoc-members:
+ :show-inheritance:
diff --git a/docs/source/api/browse.services.document.metadata.rst b/docs/source/api/browse.services.document.metadata.rst
new file mode 100644
index 000000000..40a8aae95
--- /dev/null
+++ b/docs/source/api/browse.services.document.metadata.rst
@@ -0,0 +1,7 @@
+browse.services.document.metadata module
+========================================
+
+.. automodule:: browse.services.document.metadata
+ :members:
+ :undoc-members:
+ :show-inheritance:
diff --git a/docs/source/api/browse.services.document.rst b/docs/source/api/browse.services.document.rst
new file mode 100644
index 000000000..1d76663da
--- /dev/null
+++ b/docs/source/api/browse.services.document.rst
@@ -0,0 +1,17 @@
+browse.services.document package
+================================
+
+.. automodule:: browse.services.document
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
+Subpackages
+-----------
+
+.. toctree::
+
+ browse.services.document.author_affil
+ browse.services.document.cache
+ browse.services.document.metadata
+
diff --git a/docs/source/api/browse.services.listing.fake_listings.rst b/docs/source/api/browse.services.listing.fake_listings.rst
new file mode 100644
index 000000000..16ddd84e6
--- /dev/null
+++ b/docs/source/api/browse.services.listing.fake_listings.rst
@@ -0,0 +1,7 @@
+browse.services.listing.fake_listings module
+==============================================
+
+.. automodule:: browse.services.listing.fake_listings
+ :members:
+ :undoc-members:
+ :show-inheritance:
diff --git a/docs/source/api/browse.services.listing.rst b/docs/source/api/browse.services.listing.rst
new file mode 100644
index 000000000..52011e20c
--- /dev/null
+++ b/docs/source/api/browse.services.listing.rst
@@ -0,0 +1,14 @@
+browse.services.listing package
+================================
+
+.. automodule:: browse.services.listing
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
+Subpackages
+-----------
+
+.. toctree::
+
+ browse.services.listing.fake_listings
diff --git a/docs/source/api/browse.services.rst b/docs/source/api/browse.services.rst
new file mode 100644
index 000000000..d7189959b
--- /dev/null
+++ b/docs/source/api/browse.services.rst
@@ -0,0 +1,18 @@
+browse.services package
+=======================
+
+.. automodule:: browse.services
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
+Subpackages
+-----------
+
+.. toctree::
+
+ browse.services.database
+ browse.services.document
+ browse.services.listing
+ browse.services.search
+ browse.services.util
diff --git a/docs/source/api/browse.services.search.rst b/docs/source/api/browse.services.search.rst
new file mode 100644
index 000000000..a13da02c9
--- /dev/null
+++ b/docs/source/api/browse.services.search.rst
@@ -0,0 +1,14 @@
+browse.services.search package
+================================
+
+.. automodule:: browse.services.search
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
+Subpackages
+-----------
+
+.. toctree::
+
+ browse.services.search.search_authors
diff --git a/docs/source/api/browse.services.search.search_authors.rst b/docs/source/api/browse.services.search.search_authors.rst
new file mode 100644
index 000000000..d929b7ec0
--- /dev/null
+++ b/docs/source/api/browse.services.search.search_authors.rst
@@ -0,0 +1,7 @@
+browse.services.search.search_authors module
+============================================
+
+.. automodule:: browse.services.search.search_authors
+ :members:
+ :undoc-members:
+ :show-inheritance:
diff --git a/docs/source/api/browse.services.util.email.rst b/docs/source/api/browse.services.util.email.rst
new file mode 100644
index 000000000..657e9b271
--- /dev/null
+++ b/docs/source/api/browse.services.util.email.rst
@@ -0,0 +1,7 @@
+browse.services.util.email module
+============================================
+
+.. automodule:: browse.services.util.email
+ :members:
+ :undoc-members:
+ :show-inheritance:
diff --git a/docs/source/api/browse.services.util.external_refs_cits.rst b/docs/source/api/browse.services.util.external_refs_cits.rst
new file mode 100644
index 000000000..ade155369
--- /dev/null
+++ b/docs/source/api/browse.services.util.external_refs_cits.rst
@@ -0,0 +1,7 @@
+browse.services.util.external_refs_cits module
+================================================
+
+.. automodule:: browse.services.util.external_refs_cits
+ :members:
+ :undoc-members:
+ :show-inheritance:
diff --git a/docs/source/api/browse.services.util.formats.rst b/docs/source/api/browse.services.util.formats.rst
new file mode 100644
index 000000000..fa2293015
--- /dev/null
+++ b/docs/source/api/browse.services.util.formats.rst
@@ -0,0 +1,7 @@
+browse.services.util.formats module
+================================================
+
+.. automodule:: browse.services.util.formats
+ :members:
+ :undoc-members:
+ :show-inheritance:
diff --git a/docs/source/api/browse.services.util.metatags.rst b/docs/source/api/browse.services.util.metatags.rst
new file mode 100644
index 000000000..6623c2d9b
--- /dev/null
+++ b/docs/source/api/browse.services.util.metatags.rst
@@ -0,0 +1,7 @@
+browse.services.util.metatags module
+================================================
+
+.. automodule:: browse.services.util.metatags
+ :members:
+ :undoc-members:
+ :show-inheritance:
diff --git a/docs/source/api/browse.services.util.response_headers.rst b/docs/source/api/browse.services.util.response_headers.rst
new file mode 100644
index 000000000..15daf9438
--- /dev/null
+++ b/docs/source/api/browse.services.util.response_headers.rst
@@ -0,0 +1,7 @@
+browse.services.util.response_headers module
+================================================
+
+.. automodule:: browse.services.util.response_headers
+ :members:
+ :undoc-members:
+ :show-inheritance:
diff --git a/docs/source/api/browse.services.util.rst b/docs/source/api/browse.services.util.rst
new file mode 100644
index 000000000..a20abf25c
--- /dev/null
+++ b/docs/source/api/browse.services.util.rst
@@ -0,0 +1,19 @@
+browse.services.util package
+================================
+
+.. automodule:: browse.services.util
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
+Subpackages
+-----------
+
+.. toctree::
+
+ browse.services.util.email
+ browse.services.util.external_refs_cits
+ browse.services.util.formats
+ browse.services.util.metatags
+ browse.services.util.response_headers
+ browse.services.util.tex2utf
diff --git a/docs/source/api/browse.services.util.tex2utf.rst b/docs/source/api/browse.services.util.tex2utf.rst
new file mode 100644
index 000000000..070b736dd
--- /dev/null
+++ b/docs/source/api/browse.services.util.tex2utf.rst
@@ -0,0 +1,7 @@
+browse.services.util.tex2utf module
+================================================
+
+.. automodule:: browse.services.util.tex2utf
+ :members:
+ :undoc-members:
+ :show-inheritance:
diff --git a/docs/source/api/browse.util.rst b/docs/source/api/browse.util.rst
new file mode 100644
index 000000000..2491304cb
--- /dev/null
+++ b/docs/source/api/browse.util.rst
@@ -0,0 +1,7 @@
+browse.util module
+========================================
+
+.. automodule:: browse.util
+ :members:
+ :undoc-members:
+ :show-inheritance:
diff --git a/docs/source/api/modules.rst b/docs/source/api/modules.rst
new file mode 100644
index 000000000..797a8bd73
--- /dev/null
+++ b/docs/source/api/modules.rst
@@ -0,0 +1,7 @@
+browse
+======
+
+.. toctree::
+ :maxdepth: 4
+
+ browse
\ No newline at end of file
diff --git a/docs/architecture.rst b/docs/source/architecture.rst
similarity index 99%
rename from docs/architecture.rst
rename to docs/source/architecture.rst
index a361d771d..e1445bd71 100644
--- a/docs/architecture.rst
+++ b/docs/source/architecture.rst
@@ -1,5 +1,5 @@
-Browse Application
-******************
+Browse Application Architecture
+*******************************
The browse application provides reader-facing views onto arXiv documents and
their metadata. "Browse" functionality encompasses the abstract page, content
diff --git a/docs/source/conf.py b/docs/source/conf.py
new file mode 100644
index 000000000..e941fe2ed
--- /dev/null
+++ b/docs/source/conf.py
@@ -0,0 +1,190 @@
+# -*- coding: utf-8 -*-
+#
+# Configuration file for the Sphinx documentation builder.
+#
+# This file does only contain a selection of the most common options. For a
+# full list see the documentation:
+# http://www.sphinx-doc.org/en/master/config
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+# import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+import os
+import sys
+sys.path.insert(0, os.path.abspath('.'))
+sys.path.append(os.path.abspath('..'))
+sys.path.append(os.path.abspath('../..'))
+
+# -- Project information -----------------------------------------------------
+
+project = 'arXiv Browse Service'
+copyright = '2019, arXiv-NG Team'
+author = 'arXiv-NG Team'
+
+# The short X.Y version
+version = ''
+# The full version, including alpha/beta/rc tags
+release = '0.2.1'
+
+
+# -- General configuration ---------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+ 'sphinx.ext.autodoc',
+ 'sphinx.ext.intersphinx',
+ 'sphinx.ext.coverage',
+ 'sphinx.ext.viewcode',
+ 'sphinx.ext.githubpages',
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+# source_suffix = ['.rst', '.md']
+source_suffix = '.rst'
+
+# The master toctree document.
+master_doc = 'index'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = None
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = []
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = None
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages. See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'alabaster'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further. For a list of options available for each theme, see the
+# documentation.
+#
+# html_theme_options = {}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+# Custom sidebar templates, must be a dictionary that maps document names
+# to template names.
+#
+# The default sidebars (for documents that don't match any pattern) are
+# defined by theme itself. Builtin themes are using these templates by
+# default: ``['localtoc.html', 'relations.html', 'sourcelink.html',
+# 'searchbox.html']``.
+#
+# html_sidebars = {}
+
+
+# -- Options for HTMLHelp output ---------------------------------------------
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'arXivBrowseServicedoc'
+
+
+# -- Options for LaTeX output ------------------------------------------------
+
+latex_elements = {
+ # The paper size ('letterpaper' or 'a4paper').
+ #
+ # 'papersize': 'letterpaper',
+
+ # The font size ('10pt', '11pt' or '12pt').
+ #
+ # 'pointsize': '10pt',
+
+ # Additional stuff for the LaTeX preamble.
+ #
+ # 'preamble': '',
+
+ # Latex figure (float) alignment
+ #
+ # 'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+# author, documentclass [howto, manual, or own class]).
+latex_documents = [
+ (master_doc, 'arXivBrowseService.tex', 'arXiv Browse Service Documentation',
+ 'arXiv-NG Team', 'manual'),
+]
+
+
+# -- Options for manual page output ------------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+ (master_doc, 'arxivbrowseservice', 'arXiv Browse Service Documentation',
+ [author], 1)
+]
+
+
+# -- Options for Texinfo output ----------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+# dir menu entry, description, category)
+texinfo_documents = [
+ (master_doc, 'arXivBrowseService', 'arXiv Browse Service Documentation',
+ author, 'arXivBrowseService', 'One line description of project.',
+ 'Miscellaneous'),
+]
+
+
+# -- Options for Epub output -------------------------------------------------
+
+# Bibliographic Dublin Core info.
+epub_title = project
+
+# The unique identifier of the text. This can be a ISBN number
+# or the project homepage.
+#
+# epub_identifier = ''
+
+# A unique identification for the text.
+#
+# epub_uid = ''
+
+# A list of files that should not be packed into the epub file.
+epub_exclude_files = ['search.html']
+
+
+# -- Extension configuration -------------------------------------------------
+
+# -- Options for intersphinx extension ---------------------------------------
+
+# Example configuration for intersphinx: refer to the Python standard library.
+intersphinx_mapping = {'https://docs.python.org/': None}
diff --git a/docs/source/index.rst b/docs/source/index.rst
new file mode 100644
index 000000000..9f01e357a
--- /dev/null
+++ b/docs/source/index.rst
@@ -0,0 +1,23 @@
+.. arXiv Browse Service documentation master file, created by
+ sphinx-quickstart on Wed Feb 13 16:27:27 2019.
+ You can adapt this file completely to your liking, but it should at least
+ contain the root `toctree` directive.
+
+Welcome to arXiv Browse Service's documentation!
+================================================
+
+.. toctree::
+ :maxdepth: 3
+ :caption: Contents:
+
+ architecture
+ api/modules
+
+
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`
diff --git a/tests/data/abs_files/ftp/arxiv/papers/1902/1902.11195.abs b/tests/data/abs_files/ftp/arxiv/papers/1902/1902.11195.abs
new file mode 100644
index 000000000..8e2c26b7f
--- /dev/null
+++ b/tests/data/abs_files/ftp/arxiv/papers/1902/1902.11195.abs
@@ -0,0 +1,46 @@
+------------------------------------------------------------------------------
+\\
+arXiv:1902.11195
+From: Israel Mardor
+Date: Thu, 28 Feb 2019 16:26:35 GMT (2136kb,D)
+
+Title: A Novel Method for the Measurement of Half-Lives and Decay Branching
+ Ratios of Exotic Nuclei
+Authors: Ivan Miskun (1), Timo Dickel (1 and 2), Israel Mardor (3 and 4),
+ Christine Hornung (1), Daler Amanbayev (1), Samuel Ayet San Andr\'es (1 and
+ 2), Julian Bergmann (1), Jens Ebert (1), Hans Geissel (1 and 2), Magdalena
+ G\'orska (2), Florian Greiner (1), Emma Haettner (2), Wolfgang R. Pla{\ss} (1
+ and 2), Sivaji Purushothaman (2), Christoph Scheidenberger (1 and 2),
+ Ann-Kathrin Rink (1), Helmut Weick (2), Soumya Bagchi (1 and 2 and 6), Paul
+ Constantin (5), Satbir Kaur (6), Wayne Lippert (1), Bo Mei (5), Iain Moore
+ (7), Jan-Hendrick Otto (1), Stephane Pietri (2), Ilkka Pohjalainen (7),
+ Andrej Prochazka (2), Christoph Rappold (1 and 2), Moritz P. Reiter (1 and
+ 8), Yoshiki K. Tanaka (2), John S. Winfield (2), for the Super-FRS Experiment
+ Collaboration ((1) Justus-Liebig-Universit\"at, Gie{\ss}en, Germany (2) GSI
+ Helmholtzzentrum f\"ur Schwerionenforschung GmbH, Darmstadt, Germany (3) Tel
+ Aviv University, Tel Aviv, Israel (4) Soreq Nuclear Research Center, Yavne,
+ Israel (5) IFIN-HH/ELI-NP, M\u{a}gurele - Bucharest, Romania (6) Saint Mary's
+ University, Halifax, Canada (7) University of Jyv\"askyl\"a, Jyv\"askyl\"a,
+ Finland (8) TRIUMF, Vancouver, Canada)
+Categories: nucl-ex
+Comments: 11 pages, 9 figures, 3 tables. Submitted to European Physics Journal
+ A
+License: http://arxiv.org/licenses/nonexclusive-distrib/1.0/
+\\
+ A novel method for simultaneous measurement of masses, Q-values, isomer
+excitation energies, half-lives and decay branching ratios of exotic nuclei has
+been demonstrated. The method includes first use of a stopping cell as an ion
+trap, combining containment of precursors and decay-recoils for variable
+durations in a cryogenic stopping cell (CSC), and afterwards the identification
+and counting of them by a multiple-reflection time-of-flight mass spectrometer
+(MR-TOF-MS). Feasibility has been established by recording the decay and growth
+of $^{216}$Po and $^{212}$Pb (alpha decay) and of $^{119m2}$Sb (t$_{1/2}$ =
+850$\pm$90 ms) and $^{119g}$Sb (isomer transition), obtaining half-lives and
+branching ratios consistent with literature values. Hardly any
+non-nuclear-decay losses have been observed in the CSC for up to $\sim$10
+seconds, which exhibits its extraordinary cleanliness. For $^{119}$Sb, this is
+the first direct measurement of the ground and second isomeric state masses,
+resolving the discrepancies in previous excitation energy data. These results
+pave the way for the measurement of branching ratios of exotic nuclei with
+multiple decay channels.
+\\
diff --git a/tests/data/abs_files/orig/math-ph/papers/0702031v1.abs b/tests/data/abs_files/orig/math-ph/papers/0702031v1.abs
deleted file mode 100644
index 5809d6dec..000000000
--- a/tests/data/abs_files/orig/math-ph/papers/0702031v1.abs
+++ /dev/null
@@ -1,17 +0,0 @@
-------------------------------------------------------------------------------
-\\
-arXiv:math-ph/0702031
-From: Paulo Mendon\c{c}a
-Date: Fri, 9 Feb 2007 20:39:05 GMT (8kb)
-
-Title: Probability Distribution of Curvatures of Isosurfaces in Gaussian Random
- Fields
-Authors: Paulo R. S. Mendonca, Rahul Bhotika and James V. Miller
-Categories: math-ph.MP
-Comments: Seven pages, six references
-MSC-class: 60D05
-\\
- An expression for the joint probability distribution of the principal
-curvatures at an arbitrary point in the ensemble of isosurfaces defined on
-isotropic Gaussian random fields on Rn is derived.
-\\
diff --git a/tests/data/browse.db b/tests/data/browse.db
index 3ca1a9d37..10847cc3e 100644
Binary files a/tests/data/browse.db and b/tests/data/browse.db differ
diff --git a/tests/data/daily_stats b/tests/data/daily_stats
new file mode 100644
index 000000000..31ea233b8
--- /dev/null
+++ b/tests/data/daily_stats
@@ -0,0 +1 @@
+total_papers 1234567
diff --git a/tests/data/db/sql/arXiv_in_category.sql b/tests/data/db/sql/arXiv_in_category.sql
new file mode 100644
index 000000000..8e691811a
--- /dev/null
+++ b/tests/data/db/sql/arXiv_in_category.sql
@@ -0,0 +1,9 @@
+INSERT INTO `arXiv_in_category` VALUES (1,'cond-mat','stat-mech',0),(1,'math','CO',1),(1,'math','MP',0),(1,'math-ph','',0);
+INSERT INTO `arXiv_in_category` VALUES (2,'math','AG',1),(2,'math','NT',0);
+INSERT INTO `arXiv_in_category` VALUES (3,'math','DG',1);
+INSERT INTO `arXiv_in_category` VALUES (4,'math','CT',0),(4,'math','KT',1);
+INSERT INTO `arXiv_in_category` VALUES (5,'gr-qc','',1);
+INSERT INTO `arXiv_in_category` VALUES (6,'gr-qc','',1);
+INSERT INTO `arXiv_in_category` VALUES (7,'cond-mat','mtrl-sci',0),(7,'cond-mat','supr-con',1);
+INSERT INTO `arXiv_in_category` VALUES (422353,'cs','IT',1),(422353,'math','IT',0);
+INSERT INTO `arXiv_in_category` VALUES (504934,'hep-ex','',1);
diff --git a/tests/data/db/sql/arXiv_stats_hourly.sql b/tests/data/db/sql/arXiv_stats_hourly.sql
new file mode 100644
index 000000000..9f75799cf
--- /dev/null
+++ b/tests/data/db/sql/arXiv_stats_hourly.sql
@@ -0,0 +1,5 @@
+-- full day's data for 2019-01-012
+INSERT INTO `arXiv_stats_hourly` VALUES ('2019-01-02',0,3,'N',53845),('2019-01-02',0,2,'N',48406),('2019-01-02',0,4,'N',48828),('2019-01-02',0,1,'N',51429),('2019-01-02',0,1,'A',3),('2019-01-02',1,3,'N',56725),('2019-01-02',1,2,'N',53353),('2019-01-02',1,2,'A',3),('2019-01-02',1,4,'N',48812),('2019-01-02',1,1,'N',51958),('2019-01-02',1,1,'A',5),('2019-01-02',2,3,'N',59992),('2019-01-02',2,2,'N',61270),('2019-01-02',2,1,'N',54667),('2019-01-02',2,1,'A',2),('2019-01-02',2,4,'N',50583),('2019-01-02',3,3,'N',66846),('2019-01-02',3,2,'N',63584),('2019-01-02',3,4,'N',62662),('2019-01-02',3,4,'A',2),('2019-01-02',3,1,'N',61056),('2019-01-02',3,1,'A',4),('2019-01-02',4,3,'N',59514),('2019-01-02',4,1,'N',53321),('2019-01-02',4,1,'A',2),('2019-01-02',4,2,'N',63700),('2019-01-02',4,4,'N',55601),('2019-01-02',5,3,'N',62566),('2019-01-02',5,3,'A',2),('2019-01-02',5,1,'N',57549),('2019-01-02',5,1,'A',12),('2019-01-02',5,4,'N',56604),('2019-01-02',5,2,'N',61001),('2019-01-02',5,2,'A',7),('2019-01-02',6,1,'N',56351),('2019-01-02',6,1,'A',5),('2019-01-02',6,3,'N',63059),('2019-01-02',6,4,'N',54324),('2019-01-02',6,2,'N',59236),('2019-01-02',6,2,'A',3),('2019-01-02',7,3,'N',59573),('2019-01-02',7,2,'N',61090),('2019-01-02',7,4,'N',54211),('2019-01-02',7,1,'N',57702),('2019-01-02',7,1,'A',2),('2019-01-02',8,3,'N',70691),('2019-01-02',8,2,'N',72144),('2019-01-02',8,4,'N',73142),('2019-01-02',8,1,'N',32346),('2019-01-02',8,1,'A',2),('2019-01-02',9,3,'N',65412),('2019-01-02',9,4,'N',60758),('2019-01-02',9,1,'N',56908),('2019-01-02',9,1,'A',4),('2019-01-02',9,2,'N',65283),('2019-01-02',9,2,'A',2),('2019-01-02',10,3,'N',68411),('2019-01-02',10,4,'N',60375),('2019-01-02',10,1,'N',59069),('2019-01-02',10,1,'A',2),('2019-01-02',10,2,'N',63476),('2019-01-02',11,3,'N',63408),('2019-01-02',11,3,'A',2),('2019-01-02',11,1,'N',54618),('2019-01-02',11,1,'A',19),('2019-01-02',11,4,'N',58673),('2019-01-02',11,2,'N',58308),('2019-01-02',12,3,'N',58050),('2019-01-02',12,1,'N',51337),('2019-01-02',12,1,'A',8),('2019-01-02',12,2,'N',57397),('2019-01-02',12,4,'N',54196),('2019-01-02',13,2,'N',51752),('2019-01-02',13,1,'N',45888),('2019-01-02',13,1,'A',2),('2019-01-02',13,3,'N',52890),('2019-01-02',13,4,'N',52570),('2019-01-02',14,4,'N',47099),('2019-01-02',14,1,'N',42504),('2019-01-02',14,1,'A',2),('2019-01-02',14,3,'N',49769),('2019-01-02',14,2,'N',47754),('2019-01-02',15,3,'N',51613),('2019-01-02',15,1,'N',42363),('2019-01-02',15,1,'A',6),('2019-01-02',15,2,'N',46362),('2019-01-02',15,4,'N',44812),('2019-01-02',16,3,'N',44893),('2019-01-02',16,4,'N',42667),('2019-01-02',16,2,'N',46088),('2019-01-02',16,1,'N',39196),('2019-01-02',16,1,'A',2),('2019-01-02',17,3,'N',42756),('2019-01-02',17,2,'N',44652),('2019-01-02',17,2,'A',4),('2019-01-02',17,1,'N',37109),('2019-01-02',17,1,'A',12),('2019-01-02',17,4,'N',40573),('2019-01-02',17,4,'A',5),('2019-01-02',18,3,'N',39766),('2019-01-02',18,4,'N',35739),('2019-01-02',18,4,'A',3),('2019-01-02',18,1,'N',34420),('2019-01-02',18,1,'A',5),('2019-01-02',18,2,'N',40016),('2019-01-02',19,4,'N',38157),('2019-01-02',19,3,'N',39220),('2019-01-02',19,2,'N',42100),('2019-01-02',19,1,'N',34453),('2019-01-02',19,1,'A',2),('2019-01-02',20,4,'N',53578),('2019-01-02',20,3,'N',58880),('2019-01-02',20,2,'N',61952),('2019-01-02',20,1,'N',56445),('2019-01-02',20,1,'A',2),('2019-01-02',21,4,'N',59692),('2019-01-02',21,2,'N',62295),('2019-01-02',21,2,'A',2843),('2019-01-02',21,3,'N',69497),('2019-01-02',21,1,'N',63502),('2019-01-02',21,1,'A',3116),('2019-01-02',22,4,'N',48579),('2019-01-02',22,2,'N',53989),('2019-01-02',22,3,'N',55692),('2019-01-02',22,1,'N',54425),('2019-01-02',22,1,'A',2570),('2019-01-02',23,4,'N',46401),('2019-01-02',23,4,'A',327),('2019-01-02',23,2,'N',48631),('2019-01-02',23,2,'A',1521),('2019-01-02',23,3,'N',50670),('2019-01-02',23,1,'N',51057),('2019-01-02',23,1,'A',1852);
+
+-- partial day's data for 2019-03-19
+INSERT INTO `arXiv_stats_hourly` VALUES ('2019-03-19',0,4,'N',52840),('2019-03-19',0,4,'A',3),('2019-03-19',0,2,'N',53704),('2019-03-19',0,3,'N',55367),('2019-03-19',0,3,'A',276),('2019-03-19',0,1,'N',63019),('2019-03-19',0,1,'A',6035),('2019-03-19',1,3,'N',57338),('2019-03-19',1,3,'A',667),('2019-03-19',1,2,'N',56893),('2019-03-19',1,4,'N',58308),('2019-03-19',1,4,'A',209),('2019-03-19',1,1,'N',69393),('2019-03-19',1,1,'A',7040),('2019-03-19',2,3,'N',65608),('2019-03-19',2,3,'A',282),('2019-03-19',2,2,'N',64133),('2019-03-19',2,4,'N',65662),('2019-03-19',2,4,'A',617),('2019-03-19',2,1,'N',78694),('2019-03-19',2,1,'A',3468),('2019-03-19',3,4,'N',41070),('2019-03-19',3,4,'A',163),('2019-03-19',3,2,'N',75674),('2019-03-19',3,2,'A',68),('2019-03-19',3,3,'N',74601),('2019-03-19',3,3,'A',540),('2019-03-19',3,1,'N',82732),('2019-03-19',3,1,'A',1012),('2019-03-19',4,4,'N',82631),('2019-03-19',4,4,'A',641),('2019-03-19',4,3,'N',82636),('2019-03-19',4,3,'A',709),('2019-03-19',4,2,'N',74236),('2019-03-19',4,1,'N',81817),('2019-03-19',4,1,'A',1339),('2019-03-19',5,1,'N',83531),('2019-03-19',5,1,'A',502),('2019-03-19',5,4,'N',80857),('2019-03-19',5,4,'A',303),('2019-03-19',5,3,'N',80824),('2019-03-19',5,3,'A',198),('2019-03-19',5,2,'N',78596),('2019-03-19',6,1,'N',75874),('2019-03-19',6,1,'A',977),('2019-03-19',6,3,'N',76941),('2019-03-19',6,2,'N',74382),('2019-03-19',6,4,'N',75905),('2019-03-19',7,2,'N',75759),('2019-03-19',7,1,'N',82056),('2019-03-19',7,1,'A',1908),('2019-03-19',7,3,'N',77887),('2019-03-19',7,4,'N',79405),('2019-03-19',8,1,'N',89503),('2019-03-19',8,1,'A',1975),('2019-03-19',8,2,'N',79577),('2019-03-19',8,4,'N',88666),('2019-03-19',8,3,'N',84634);
diff --git a/tests/data/db/sql/arXiv_stats_monthly_downloads.sql b/tests/data/db/sql/arXiv_stats_monthly_downloads.sql
new file mode 100644
index 000000000..eb9d7f18f
--- /dev/null
+++ b/tests/data/db/sql/arXiv_stats_monthly_downloads.sql
@@ -0,0 +1 @@
+INSERT INTO `arXiv_stats_monthly_downloads` VALUES ('1994-01-01',0),('1994-02-01',1863),('1994-03-01',7048),('1994-04-01',6814),('1994-05-01',9601),('1994-06-01',11007),('1994-07-01',9488),('1994-08-01',10789),('1994-09-01',12872),('1994-10-01',16260),('1994-11-01',22392),('1994-12-01',20070),('1995-01-01',24801),('1995-02-01',28607),('1995-03-01',35418),('1995-04-01',34109),('1995-05-01',35232),('1995-06-01',52415),('1995-07-01',68341),('1995-08-01',95848),('1995-09-01',114550),('1995-10-01',146524),('1995-11-01',151624),('1995-12-01',132908),('1996-01-01',160742),('1996-02-01',186528),('1996-03-01',193169),('1996-04-01',198175),('1996-05-01',221716),('1996-06-01',213072),('1996-07-01',230395),('1996-08-01',222063),('1996-09-01',251249),('1996-10-01',287532),('1996-11-01',263905),('1996-12-01',225928),('1997-01-01',250816),('1997-02-01',230357),('1997-03-01',239669),('1997-04-01',258302),('1997-05-01',267683),('1997-06-01',261706),('1997-07-01',270550),('1997-08-01',239698),('1997-09-01',287158),('1997-10-01',326150),('1997-11-01',296375),('1997-12-01',288487),('1998-01-01',321044),('1998-02-01',337514),('1998-03-01',407658),('1998-04-01',384583),('1998-05-01',409663),('1998-06-01',404232),('1998-07-01',412267),('1998-08-01',382223),('1998-09-01',444628),('1998-10-01',510790),('1998-11-01',537888),('1998-12-01',474617),('1999-01-01',508224),('1999-02-01',464452),('1999-03-01',511665),('1999-04-01',510014),('1999-05-01',536821),('1999-06-01',547141),('1999-07-01',544712),('1999-08-01',491688),('1999-09-01',568449),('1999-10-01',582934),('1999-11-01',665411),('1999-12-01',627284),('2000-01-01',640945),('2000-02-01',725616),('2000-03-01',751154),('2000-04-01',697884),('2000-05-01',785650),('2000-06-01',745325),('2000-07-01',728494),('2000-08-01',721467),('2000-09-01',724899),('2000-10-01',828870),('2000-11-01',925260),('2000-12-01',844998),('2001-01-01',935314),('2001-02-01',972075),('2001-03-01',1055737),('2001-04-01',1013386),('2001-05-01',1059624),('2001-06-01',902582),('2001-07-01',846039),('2001-08-01',865395),('2001-09-01',834963),('2001-10-01',969584),('2001-11-01',966553),('2001-12-01',1145430),('2002-01-01',1034782),('2002-02-01',982129),('2002-03-01',1175829),('2002-04-01',1321376),('2002-05-01',1423935),('2002-06-01',1236836),('2002-07-01',1348311),('2002-08-01',1270201),('2002-09-01',1516847),('2002-10-01',1675586),('2002-11-01',1515041),('2002-12-01',1343960),('2003-01-01',1588531),('2003-02-01',1572793),('2003-03-01',1711279),('2003-04-01',1620843),('2003-05-01',1703389),('2003-06-01',1601569),('2003-07-01',1586144),('2003-08-01',1504793),('2003-09-01',1820701),('2003-10-01',2117036),('2003-11-01',2033657),('2003-12-01',2119846),('2004-01-01',2109079),('2004-02-01',2293511),('2004-03-01',2221885),('2004-04-01',2100595),('2004-05-01',2301904),('2004-06-01',2036838),('2004-07-01',1986919),('2004-08-01',1997330),('2004-09-01',2191867),('2004-10-01',1927789),('2004-11-01',1978766),('2004-12-01',1767095),('2005-01-01',1804435),('2005-02-01',1803019),('2005-03-01',2068684),('2005-04-01',2251800),('2005-05-01',2136947),('2005-06-01',1951444),('2005-07-01',1852432),('2005-08-01',1852131),('2005-09-01',2120662),('2005-10-01',2346256),('2005-11-01',2406316),('2005-12-01',2360494),('2006-01-01',1701784),('2006-02-01',1737794),('2006-03-01',2298683),('2006-04-01',1997694),('2006-05-01',2214161),('2006-06-01',1801721),('2006-07-01',1401479),('2006-08-01',1629104),('2006-09-01',1768658),('2006-10-01',1982461),('2006-11-01',1930368),('2006-12-01',1663348),('2007-01-01',2029217),('2007-02-01',2066818),('2007-03-01',2134854),('2007-04-01',1979950),('2007-05-01',2224762),('2007-06-01',2224194),('2007-07-01',2051276),('2007-08-01',1822372),('2007-09-01',1998723),('2007-10-01',2446094),('2007-11-01',2531762),('2007-12-01',2159429),('2008-01-01',2665839),('2008-02-01',2557077),('2008-03-01',2848757),('2008-04-01',2810180),('2008-05-01',2958761),('2008-06-01',2706793),('2008-07-01',2730226),('2008-08-01',2450019),('2008-09-01',3278451),('2008-10-01',2937969),('2008-11-01',3066092),('2008-12-01',2638793),('2009-01-01',2598159),('2009-02-01',2644797),('2009-03-01',3104692),('2009-04-01',2837273),('2009-05-01',2877553),('2009-06-01',2792334),('2009-07-01',2667955),('2009-08-01',2538405),('2009-09-01',2907441),('2009-10-01',3167801),('2009-11-01',2912668),('2009-12-01',2723394),('2010-01-01',3116867),('2010-02-01',2992581),('2010-03-01',3459004),('2010-04-01',3292459),('2010-05-01',3293995),('2010-06-01',3335937),('2010-07-01',3098569),('2010-08-01',2879133),('2010-09-01',3144001),('2010-10-01',3684560),('2010-11-01',3829250),('2010-12-01',3775736),('2011-01-01',3439915),('2011-02-01',3339054),('2011-03-01',4117851),('2011-04-01',3659749),('2011-05-01',3997359),('2011-06-01',3707592),('2011-07-01',3771042),('2011-08-01',3774913),('2011-09-01',4973988),('2011-10-01',5563412),('2011-11-01',4962036),('2011-12-01',4433059),('2012-01-01',4796243),('2012-02-01',5078432),('2012-03-01',5516536),('2012-04-01',5423005),('2012-05-01',5067807),('2012-06-01',5073183),('2012-07-01',5624113),('2012-08-01',6040066),('2012-09-01',5372689),('2012-10-01',5904272),('2012-11-01',5373061),('2012-12-01',4549990),('2013-01-01',5665096),('2013-02-01',5136321),('2013-03-01',6121433),('2013-04-01',5552584),('2013-05-01',6184081),('2013-06-01',5033496),('2013-07-01',5251361),('2013-08-01',4985314),('2013-09-01',5319428),('2013-10-01',6146267),('2013-11-01',5953050),('2013-12-01',5464673),('2014-01-01',6701768),('2014-02-01',6175371),('2014-03-01',7371741),('2014-04-01',7207160),('2014-05-01',6814576),('2014-06-01',6650843),('2014-07-01',6934640),('2014-08-01',6368537),('2014-09-01',7129287),('2014-10-01',9150599),('2014-11-01',10771479),('2014-12-01',9877539),('2015-01-01',10211128),('2015-02-01',10375127),('2015-03-01',12826855),('2015-04-01',11289672),('2015-05-01',11182540),('2015-06-01',12568030),('2015-07-01',9585300),('2015-08-01',9180739),('2015-09-01',10741431),('2015-10-01',13882451),('2015-11-01',14200441),('2015-12-01',10286698),('2016-01-01',16227193),('2016-02-01',14547141),('2016-03-01',16144490),('2016-04-01',14330220),('2016-05-01',14042457),('2016-06-01',14344946),('2016-07-01',12885660),('2016-08-01',11795801),('2016-09-01',13958435),('2016-10-01',11202303),('2016-11-01',11492492),('2016-12-01',12136801),('2017-01-01',12150875),('2017-02-01',12870204),('2017-03-01',14319435),('2017-04-01',15628477),('2017-05-01',14894937),('2017-06-01',14372378),('2017-07-01',14808475),('2017-08-01',15915733),('2017-09-01',16999890),('2017-10-01',19433648),('2017-11-01',18885644),('2017-12-01',17361253),('2018-01-01',18445580),('2018-02-01',17704444),('2018-03-01',20149872),('2018-04-01',19709932),('2018-05-01',19710718),('2018-06-01',18088369),('2018-07-01',18132320),('2018-08-01',17906302),('2018-09-01',17418053),('2018-10-01',19443085),('2018-11-01',23040835),('2018-12-01',18694165),('2019-01-01',19281479);
diff --git a/tests/data/db/sql/arXiv_stats_monthly_submissions.sql b/tests/data/db/sql/arXiv_stats_monthly_submissions.sql
new file mode 100644
index 000000000..23932c0d6
--- /dev/null
+++ b/tests/data/db/sql/arXiv_stats_monthly_submissions.sql
@@ -0,0 +1 @@
+INSERT INTO `arXiv_stats_monthly_submissions` VALUES ('1991-07-01',2,-2),('1991-08-01',28,-1),('1991-09-01',58,0),('1991-10-01',76,0),('1991-11-01',64,0),('1991-12-01',78,0),('1992-01-01',193,-105),('1992-02-01',134,-10),('1992-03-01',120,-3),('1992-04-01',225,-41),('1992-05-01',237,-11),('1992-06-01',237,-5),('1992-07-01',296,-22),('1992-08-01',227,-4),('1992-09-01',336,-21),('1992-10-01',401,-29),('1992-11-01',453,-16),('1992-12-01',404,-10),('1993-01-01',400,-36),('1993-02-01',428,-16),('1993-03-01',504,-9),('1993-04-01',505,-17),('1993-05-01',538,-7),('1993-06-01',536,-16),('1993-07-01',643,-35),('1993-08-01',541,-25),('1993-09-01',518,-16),('1993-10-01',678,-36),('1993-11-01',714,-16),('1993-12-01',738,-15),('1994-01-01',610,-28),('1994-02-01',653,-19),('1994-03-01',753,-24),('1994-04-01',738,-39),('1994-05-01',841,-18),('1994-06-01',888,-32),('1994-07-01',863,-23),('1994-08-01',766,-20),('1994-09-01',884,-33),('1994-10-01',932,-23),('1994-11-01',1114,-39),('1994-12-01',1055,-33),('1995-01-01',933,-36),('1995-02-01',977,-36),('1995-03-01',1168,-37),('1995-04-01',913,-31),('1995-05-01',1110,-22),('1995-06-01',1195,-27),('1995-07-01',1017,-25),('1995-08-01',1030,-22),('1995-09-01',1133,-29),('1995-10-01',1229,-23),('1995-11-01',1162,-28),('1995-12-01',1147,-37),('1996-01-01',1051,-22),('1996-02-01',1076,-23),('1996-03-01',1169,-23),('1996-04-01',1220,-49),('1996-05-01',1358,-42),('1996-06-01',1311,-34),('1996-07-01',1424,-32),('1996-08-01',1394,-19),('1996-09-01',1474,-21),('1996-10-01',1520,-30),('1996-11-01',1402,-27),('1996-12-01',1467,-33),('1997-01-01',1308,-31),('1997-02-01',1362,-34),('1997-03-01',1411,-24),('1997-04-01',1468,-34),('1997-05-01',1581,-32),('1997-06-01',1709,-31),('1997-07-01',1804,-41),('1997-08-01',1426,-15),('1997-09-01',1882,-37),('1997-10-01',2007,-26),('1997-11-01',1717,-32),('1997-12-01',1949,-31),('1998-01-01',1723,-11),('1998-02-01',1668,0),('1998-03-01',1911,-3),('1998-04-01',1747,-12),('1998-05-01',1918,-2),('1998-06-01',2089,-6),('1998-07-01',2101,-12),('1998-08-01',1826,-6),('1998-09-01',2414,-9),('1998-10-01',2330,-33),('1998-11-01',2242,-14),('1998-12-01',2203,-7),('1999-01-01',1850,-23),('1999-02-01',1919,-3),('1999-03-01',2394,-15),('1999-04-01',2156,-9),('1999-05-01',2221,-17),('1999-06-01',2427,-5),('1999-07-01',2414,-23),('1999-08-01',2139,-1),('1999-09-01',2502,-12),('1999-10-01',2571,-6),('1999-11-01',2518,-29),('1999-12-01',2593,-6),('2000-01-01',2365,-6),('2000-02-01',2366,0),('2000-03-01',2600,-9),('2000-04-01',2077,-9),('2000-05-01',2727,-9),('2000-06-01',2439,-3),('2000-07-01',2461,-16),('2000-08-01',2613,-1),('2000-09-01',2522,-7),('2000-10-01',2913,-3),('2000-11-01',2864,-11),('2000-12-01',2654,-1),('2001-01-01',2591,-14),('2001-02-01',2429,-3),('2001-03-01',2713,-8),('2001-04-01',2606,-6),('2001-05-01',2912,-4),('2001-06-01',2871,-5),('2001-07-01',2760,-3),('2001-08-01',2422,0),('2001-09-01',2548,-3),('2001-10-01',3404,-4),('2001-11-01',3240,-1),('2001-12-01',2718,-19),('2002-01-01',2716,-4),('2002-02-01',2567,-2),('2002-03-01',2666,-3),('2002-04-01',2853,-4),('2002-05-01',3089,-2),('2002-06-01',2695,0),('2002-07-01',3269,-6),('2002-08-01',2725,0),('2002-09-01',3314,-2),('2002-10-01',3531,0),('2002-11-01',3440,-3),('2002-12-01',3256,-35),('2003-01-01',2926,0),('2003-02-01',2881,0),('2003-03-01',3018,0),('2003-04-01',3102,0),('2003-05-01',3267,0),('2003-06-01',3463,0),('2003-07-01',3418,0),('2003-08-01',2741,0),('2003-09-01',3747,-12),('2003-10-01',3841,0),('2003-11-01',3349,0),('2003-12-01',3661,0),('2004-01-01',3071,0),('2004-02-01',3277,-1),('2004-03-01',3603,0),('2004-04-01',3369,-1),('2004-05-01',3553,0),('2004-06-01',3710,0),('2004-07-01',3679,0),('2004-08-01',3294,0),('2004-09-01',3939,0),('2004-10-01',4086,0),('2004-11-01',4152,0),('2004-12-01',3994,0),('2005-01-01',3493,0),('2005-02-01',3250,0),('2005-03-01',3881,0),('2005-04-01',3693,0),('2005-05-01',3779,0),('2005-06-01',3984,0),('2005-07-01',3839,0),('2005-08-01',3775,0),('2005-09-01',4345,-16),('2005-10-01',4439,0),('2005-11-01',4294,0),('2005-12-01',4083,0),('2006-01-01',3858,0),('2006-02-01',3520,0),('2006-03-01',4213,0),('2006-04-01',3474,0),('2006-05-01',4204,0),('2006-06-01',4139,0),('2006-07-01',4197,0),('2006-08-01',4064,0),('2006-09-01',4275,0),('2006-10-01',5133,0),('2006-11-01',4854,0),('2006-12-01',4296,0),('2007-01-01',4653,0),('2007-02-01',4164,0),('2007-03-01',4493,0),('2007-04-01',4003,0),('2007-05-01',4684,0),('2007-06-01',4484,0),('2007-07-01',4681,0),('2007-08-01',4414,0),('2007-09-01',4682,0),('2007-10-01',5945,0),('2007-11-01',5029,0),('2007-12-01',4406,0),('2008-01-01',4970,0),('2008-02-01',4463,0),('2008-03-01',4519,0),('2008-04-01',4898,0),('2008-05-01',4836,0),('2008-06-01',4980,0),('2008-07-01',5139,0),('2008-08-01',4160,0),('2008-09-01',5287,0),('2008-10-01',5773,0),('2008-11-01',4774,0),('2008-12-01',5116,0),('2009-01-01',4975,0),('2009-02-01',4903,0),('2009-03-01',5547,0),('2009-04-01',4930,0),('2009-05-01',4955,0),('2009-06-01',5614,0),('2009-07-01',5606,0),('2009-08-01',4597,0),('2009-09-01',5696,0),('2009-10-01',5957,0),('2009-11-01',5730,0),('2009-12-01',5537,0),('2010-01-01',5471,0),('2010-02-01',5048,0),('2010-03-01',6133,0),('2010-04-01',5602,0),('2010-05-01',5737,0),('2010-06-01',5963,0),('2010-07-01',5521,0),('2010-08-01',5399,0),('2010-09-01',6232,0),('2010-10-01',6304,0),('2010-11-01',6676,0),('2010-12-01',6045,0),('2011-01-01',6081,0),('2011-02-01',5774,0),('2011-03-01',6286,0),('2011-04-01',5711,0),('2011-05-01',6374,0),('2011-06-01',6357,0),('2011-07-01',6046,0),('2011-08-01',6331,0),('2011-09-01',6937,0),('2011-10-01',6930,0),('2011-11-01',7319,0),('2011-12-01',6432,0),('2012-01-01',6687,0),('2012-02-01',6685,0),('2012-03-01',6903,0),('2012-04-01',6739,0),('2012-05-01',7092,0),('2012-06-01',7121,0),('2012-07-01',7358,0),('2012-08-01',6592,0),('2012-09-01',6630,0),('2012-10-01',8452,0),('2012-11-01',7370,0),('2012-12-01',6974,0),('2013-01-01',7750,0),('2013-02-01',7317,0),('2013-03-01',7476,0),('2013-04-01',8135,0),('2013-05-01',7516,0),('2013-06-01',6944,0),('2013-07-01',8447,0),('2013-08-01',6833,0),('2013-09-01',7995,0),('2013-10-01',8658,0),('2013-11-01',7692,0),('2013-12-01',7878,0),('2014-01-01',8294,0),('2014-02-01',7374,0),('2014-03-01',8154,0),('2014-04-01',7855,0),('2014-05-01',7975,0),('2014-06-01',7874,0),('2014-07-01',8549,0),('2014-08-01',7119,0),('2014-09-01',8676,0),('2014-10-01',8871,0),('2014-11-01',8006,0),('2014-12-01',8770,0),('2015-01-01',7912,0),('2015-02-01',8054,0),('2015-03-01',9191,0),('2015-04-01',8367,0),('2015-05-01',8172,0),('2015-06-01',9217,0),('2015-07-01',8995,0),('2015-08-01',7983,0),('2015-09-01',9318,0),('2015-10-01',9223,0),('2015-11-01',9472,0),('2015-12-01',9376,0),('2016-01-01',8251,0),('2016-02-01',9142,0),('2016-03-01',9746,0),('2016-04-01',8948,0),('2016-05-01',9792,0),('2016-06-01',9644,0),('2016-07-01',8911,0),('2016-08-01',9016,0),('2016-09-01',9869,0),('2016-10-01',10100,0),('2016-11-01',10362,0),('2016-12-01',9599,0),('2017-01-01',9186,0),('2017-02-01',8910,0),('2017-03-01',11008,0),('2017-04-01',9029,0),('2017-05-01',11194,0),('2017-06-01',10297,0),('2017-07-01',9980,0),('2017-08-01',9854,0),('2017-09-01',10517,0),('2017-10-01',11627,0),('2017-11-01',11589,0),('2017-12-01',10332,0),('2018-01-01',10609,0),('2018-02-01',10593,0),('2018-03-01',11560,0),('2018-04-01',11352,0),('2018-05-01',12595,0),('2018-06-01',11568,0),('2018-07-01',11938,0),('2018-08-01',10870,0),('2018-09-01',11171,0),('2018-10-01',13446,0),('2018-11-01',12941,0),('2018-12-01',11973,0),('2019-01-01',11537,0);
diff --git a/tests/data/db/sql/arXiv_trackback_pings.sql b/tests/data/db/sql/arXiv_trackback_pings.sql
index ba0b01357..8fd0575f8 100644
--- a/tests/data/db/sql/arXiv_trackback_pings.sql
+++ b/tests/data/db/sql/arXiv_trackback_pings.sql
@@ -1,3 +1,4 @@
+-- 0808.4142
INSERT INTO `arXiv_trackback_pings` VALUES (695736,504934,'Omega b: the new baryon nailed by D0 « A Quantum Diaries Survivor',' pulling this new result off! You can read the details of the analysis in this paper. ','http://dorigo.wordpress.com/2008/09/19/omega-b-the-new-baryon-nailed-by-d0/','','arxiv1.library.cornell.edu','128.84.158.114',1221882921,0,0,1221883202,'accepted',61);
INSERT INTO `arXiv_trackback_pings` VALUES (751383,504934,'Nit-Picking On The Omega_b Discovery','quark and two strange quarks- has been recently published by the DZERO collaboration. Their paper claims to observe the so-far-unseen','http://www.scientificblogging.com/quantum_diaries_survivor/nitpicking_omega_b_discovery','A Quantum Diaries Survivor','arxiv1.library.cornell.edu','128.84.158.114',1243482527,0,0,1243484559,'accepted',NULL);
INSERT INTO `arXiv_trackback_pings` VALUES (751384,504934,'Nit-Picking On The Omega_b Baryon: Part II','the pains to check the calculation hinted at on the scientific paper where the discovery is claimed, which was sent to the ','http://www.scientificblogging.com/quantum_diaries_survivor/nitpicking_omega_b_baryon_part_ii','A Quantum Diaries Survivor','arxiv1.library.cornell.edu','128.84.158.114',1243482588,0,0,1243484559,'accepted',NULL);
@@ -12,5 +13,4 @@ INSERT INTO `arXiv_trackback_pings` VALUES (1,504934,'More On The Omega B Baryon
-- This next one is an intentional duplicate, but with different url, and not accepted
INSERT INTO `arXiv_trackback_pings` VALUES (2,504934,'Foo baz','\"Observation of the','http://en.wikipedia.org/wiki/Zevatron','Wikipedia','arxiv1.library.cornell.edu','128.84.158.114',1228752458,0,0,0,'rejected',NULL);
-INSERT INTO `arXiv_trackback_pings` VALUES (99999999,99999998,'Nothing here','Really nothing here ','http://www.example.org','On Nothing','arxiv1.library.cornell.edu','128.84.158.114',1258488297,0,0,1248558336,'accepted',NULL);
-
+INSERT INTO `arXiv_trackback_pings` VALUES (99999999,504934,'Nothing here','Really nothing here ','http://www.example.org','On Nothing','arxiv1.library.cornell.edu','128.84.158.114',1258488294,0,0,1248558336,'accepted',NULL);
diff --git a/tests/legacy_comparison/archive_config.py b/tests/legacy_comparison/archive_config.py
new file mode 100644
index 000000000..964ccf083
--- /dev/null
+++ b/tests/legacy_comparison/archive_config.py
@@ -0,0 +1,12 @@
+from response_comparisons import compare_status
+
+res_comparisons = [ compare_status ]
+text_comparisons = []
+html_comparisons = []
+
+def ng_id_to_url_fn(id:str)->str:
+ return 'http://localhost:5000/archive/' + id
+
+def legacy_id_to_url_fn(id:str)->str:
+ return 'https://arxiv.org/archive/' + id
+
diff --git a/tests/legacy_comparison/archives.txt b/tests/legacy_comparison/archives.txt
new file mode 100644
index 000000000..df3f31942
--- /dev/null
+++ b/tests/legacy_comparison/archives.txt
@@ -0,0 +1,20 @@
+astro-ph
+cond-mat
+gr-qc
+hep-ex
+hep-lat
+hep-ph
+hep-th
+math-ph
+nlin
+nucl-ex
+nucl-th
+physics
+quant-ph
+math
+cs
+q-bio
+q-fin
+stat
+eess
+econ
diff --git a/tests/legacy_comparison/html_comparisons.py b/tests/legacy_comparison/html_comparisons.py
index 9d7a5eda7..f130db375 100644
--- a/tests/legacy_comparison/html_comparisons.py
+++ b/tests/legacy_comparison/html_comparisons.py
@@ -1,10 +1,10 @@
-from functools import partial
+from functools import partial, update_wrapper
from typing import Callable, List
import re
-from tests.legacy_comparison.abstract_comparisons import lev_similarity
-from tests.legacy_comparison.comparison_types import html_arg_dict, BadResult
+from abstract_comparisons import lev_similarity
+from comparison_types import html_arg_dict, BadResult
from bs4 import BeautifulSoup, element
@@ -15,7 +15,9 @@ def html_similarity(html_arg: html_arg_dict) -> BadResult:
html_arg['legacy_html'].prettify()
)
if sim < 0.69:
- return BadResult(html_arg['paper_id'], f"html_pretty_sim for {html_arg['paper_id']} = {sim}")
+ return BadResult(html_arg['id'],
+ "html_similarity",
+ f"html_pretty_sim for {html_arg['id']} = {sim}")
return None
@@ -34,7 +36,7 @@ def to_label(tr): return tr.find('td', 'label').contents
if ng_labels == legacy_labels:
return None
else:
- return BadResult(html_arg['paper_id'],
+ return BadResult(html_arg['id'],
"Metadata field included on NG do not match those from legacy" +
f"NG: {ng_labels} Legacy: {legacy_labels}")
@@ -69,14 +71,14 @@ def _element_similarity(name: str,
if required:
if len(ng) == 0 and len(legacy) == 0:
- return BadResult(html_arg['paper_id'], name,
- f"Missing field {name} for {html_arg['paper_id']} from NG and Legacy")
+ return BadResult(html_arg['id'], name,
+ f"Missing field {name} for {html_arg['id']} from NG and Legacy")
if len(ng) == 0:
- return BadResult(html_arg['paper_id'], name,
- f"Missing field {name} for {html_arg['paper_id']} from NG")
+ return BadResult(html_arg['id'], name,
+ f"Missing field {name} for {html_arg['id']} from NG")
if len(legacy) == 0:
- return BadResult(html_arg['paper_id'], name,
- f"Missing field {name} for {html_arg['paper_id']} from legacy")
+ return BadResult(html_arg['id'], name,
+ f"Missing field {name} for {html_arg['id']} from legacy")
if check_counts and (len(legacy) != len(ng)):
if ng:
@@ -88,8 +90,8 @@ def _element_similarity(name: str,
else:
legacy_ele_txt = 'MISSING'
- return BadResult(html_arg['paper_id'], name,
- f"bad counts for {name} for {html_arg['paper_id']} ng: {len(ng)} legacy: {len(legacy)}",
+ return BadResult(html_arg['id'], name,
+ f"bad counts for {name} for {html_arg['id']} ng: {len(ng)} legacy: {len(legacy)}",
legacy_ele_txt, ng_ele_txt)
ng_ele_txt = ''
@@ -102,9 +104,11 @@ def _element_similarity(name: str,
if sim < min_sim:
msg = f"Elements did not meet min similarity of {min_sim}"
- return BadResult(html_arg['paper_id'], name, msg, legacy_ele_txt,
+ return BadResult(html_arg['id'], name, msg, legacy_ele_txt,
ng_ele_txt, sim)
- return None
+
+ msg = f"GOOD: Elements did meet min similarity of {min_sim}"
+ return BadResult(html_arg['id'], name, msg, '','', sim)
else:
if not required:
return None
@@ -116,7 +120,7 @@ def _element_similarity(name: str,
msg = 'zero elements detected: ' \
+ f'legacy length was {len(legacy)}; ng length was {len(ng)} '
- return BadResult(html_arg['paper_id'], name, msg, legacy_ele_txt,
+ return BadResult(html_arg['id'], name, msg, legacy_ele_txt,
ng_ele_txt, 0.0)
@@ -197,3 +201,32 @@ def ex_strip(eles: List[BeautifulSoup]):
bookmarks_similarity = partial(_element_similarity, 'extra bookmarks div',
lambda bs: ex_strip(bs.select('.bookmarks')),
0.9, False, True)
+
+################# /archive checks ################################
+
+archive_h1_similarity = partial(_element_similarity, 'top heading',
+ lambda bs: ex_strip(bs.select('#content > h1')),
+ 0.99, True, True)
+
+archive_browse = partial(_element_similarity, 'browse',
+ lambda bs: ex_strip(bs.select('#content > ul > li:nth-child(1)')),
+ 0.99, True, True)
+
+archive_catchup = partial(_element_similarity, 'archive catchup',
+ lambda bs: ex_strip(bs.select('#content > ul > li:nth-child(2)')),
+ 0.99, True, True)
+
+archive_search= partial(_element_similarity, 'archive_search',
+ lambda bs: ex_strip(bs.select('#content > ul > li:nth-child(3)')),
+ 0.99, True, True)
+
+archive_by_year= partial(_element_similarity, 'archive_by_year',
+ lambda bs: ex_strip(bs.select('#content > ul > li:nth-child(4)')),
+ 0.99, True, True)
+
+
+archive_bogus= partial(_element_similarity, 'bogus_should_fail',
+ lambda bs: ex_strip(bs.select('.bogusClass')),
+ 0.99, True, True)
+
+
diff --git a/tests/legacy_comparison/page_comparison.py b/tests/legacy_comparison/page_comparison.py
new file mode 100644
index 000000000..de577dad1
--- /dev/null
+++ b/tests/legacy_comparison/page_comparison.py
@@ -0,0 +1,295 @@
+import argparse
+import itertools
+import sys
+import traceback
+import os
+import re
+from functools import partial
+import multiprocessing_on_dill as mp
+from typing import Callable, Iterator, List, Set, Tuple, Dict, Any
+import gzip
+import logging
+import json
+
+import requests
+from bs4 import BeautifulSoup
+
+# BDC34: some how I need this under pipenv to get to browse, not sure why
+sys.path.append('')
+sys.setrecursionlimit(10000)
+
+from comparison_types import res_comparison_fn, \
+ text_comparison_fn, html_comparison_fn, res_arg_dict, text_arg_dict, \
+ html_arg_dict, BadResult
+from html_comparisons import html_similarity, metadata_fields_similarity, \
+ archive_h1_similarity,archive_catchup,archive_search,archive_by_year,archive_browse,\
+ archive_bogus
+
+from response_comparisons import compare_status
+from text_comparisons import text_similarity
+
+import archive_config
+
+
+""" Script to compare pages from NG and beta.arxiv.org
+
+To run this I do:
+Open terminal:
+cd arxiv-browse
+pipenv sync
+FLASK_APP=app.py FLASK_DEBUG=1 pipenv run flask run
+
+In another terminal:
+cd arxiv-browse
+pipenv sync
+pipenv shell
+python tests/legacy_comparison/page_comparison.py
+
+To reset the analysis to start over, add the `--reset` arg.
+To run a short test add '--short' arg.
+
+To skip ancillary file comparisons: '--skip-ancillary'
+
+Improvements:
+ Better reporting format, right now the comparisons produce just strings.
+"""
+
+logging.basicConfig(filename="page_comparison.log", level=logging.DEBUG)
+
+#This just renames None nicely, they are tests that passed
+SUCCESS = None
+
+LOG_FILE_NAME = 'legacy_comparison.org'
+
+
+# TODO abstract this or move this out
+VISITED_FILE_NAME = 'visited.log'
+
+def ident(x):
+ return x
+
+configs = {
+ 'archive': { # I'd like to do something with modules but it doesn't pickle.
+ 'comparisons': [compare_status,
+ html_similarity,
+ archive_h1_similarity,
+ archive_browse,
+ archive_catchup,
+ archive_search,
+ archive_by_year,
+ ],
+ 'ng_id_to_url_fn': archive_config.ng_id_to_url_fn,
+ 'legacy_id_to_url_fn': archive_config.legacy_id_to_url_fn,
+ 'ng_txt_trans_fn': ident,
+ 'legacy_txt_trans_fn': ident,
+ }
+}
+
+# id file is one id per line
+def _id_generator_from_file(path: str, excluded: List[str])->Iterator[str]:
+ if 'gzip' in path or 'gz' in path:
+ with gzip.open(path, 'rt') as f:
+ for line in f:
+ aid = line.strip()
+ if aid not in excluded:
+ logging.debug(f'yielding id {aid}')
+ yield aid
+ else:
+ with open(path, 'rt') as f:
+ for line in f:
+ aid = line.strip()
+ if aid not in excluded:
+ logging.debug(f'yielding id {aid}')
+ yield aid
+
+#TODO generalize
+# Should end with /
+#ng_abs_base_url = 'http://localhost:5000/abs/'
+#ng_abs_base_url = 'https://beta.arxiv.org/abs/'
+
+#TODO generalize
+# Should end with /
+#legacy_abs_base_url = 'https://beta.arxiv.org/abs_classic/'
+
+def fetch_pages(config: Dict, id: str) ->Dict:
+ """Fetch NG and Legacy."""
+ ng_url = config['ng_id_to_url_fn'](id)
+ legacy_url = config['legacy_id_to_url_fn'](id)
+
+ headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'}
+ ng_res = requests.get(ng_url, headers=headers)
+ legacy_res = requests.get(legacy_url, headers=headers)
+
+ ng_text = config['ng_txt_trans_fn'](ng_res.text)
+ legacy_text = config['ng_txt_trans_fn'](legacy_res.text)
+
+ res_dict: res_arg_dict = {
+ 'id': id,
+ 'ng_url': ng_url,
+ 'legacy_url': legacy_url,
+ }
+
+ # to avoid json problems
+ with_html = {
+ **res_dict,
+ 'ng_res': ng_res,
+ 'legacy_res': legacy_res,
+ 'ng_text': ng_text,
+ 'legacy_text': legacy_text,
+ 'ng_html': BeautifulSoup(ng_text, 'html.parser'),
+ 'legacy_html': BeautifulSoup(legacy_text, 'html.parser'),
+ }
+ return (res_dict, compare_response(config,
+ with_html))
+
+
+def compare_response(config: Dict,
+ res_args: res_arg_dict) -> Iterator[BadResult]:
+ """Do the response comparisions, the kick off the text comparisions."""
+# protected_comps = [protect(fn) for fn in config['comparisons']]
+ protected_comps = config['comparisons']
+ logging.debug(f"about to do compares for {res_args['id']}")
+ return filter(SUCCESS, [fn(res_args) for fn in protected_comps])
+
+
+def multi_ws_to_single_ws(txt:str)->str:
+ return ' '.join(txt.split()) #white space to single spaces
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser(description='Compare ng pages to legacy pages')
+ parser.add_argument('--idfile', default=False, )
+ parser.add_argument('--reset', default=False, const=True,
+ action='store_const', dest='reset')
+ parser.add_argument('--short', default=False, const=True,
+ action='store_const', dest='short')
+ parser.add_argument('--config', default=False)
+ args = parser.parse_args()
+
+ print('Starting config')
+ if args.config not in configs.keys():
+ raise ValueError(f"No config named '{args.config}' choose one of [{' '.join(configs.keys())}]")
+ else:
+ print(f'Using config {args.config}')
+ active_config = configs[args.config]
+ print('done with config')
+
+ visited: Set[str] = []
+ if args.reset:
+ print('Restarting analysis and deleting logs!')
+ if os.path.exists(LOG_FILE_NAME):
+ os.remove(LOG_FILE_NAME)
+ if os.path.exists(VISITED_FILE_NAME):
+ os.remove(VISITED_FILE_NAME)
+ else:
+ if os.path.exists(VISITED_FILE_NAME):
+ print('Continuing analysis')
+ with open(VISITED_FILE_NAME, 'r') as visited_fh:
+ visited = {line.rstrip() for line in visited_fh.readlines()}
+
+ ids = _id_generator_from_file(args.idfile, excluded=visited)
+
+ if args.short:
+ n=0
+ total = 10
+ logging.info(f'Doing short list of {n}')
+ def done()->bool:
+ nonlocal n
+ if n >= total:
+ return True
+ n = n + 1
+ return False
+ else:
+ def done()->bool:
+ return False
+
+ f_then_c = partial( fetch_pages, active_config)
+
+ with open(VISITED_FILE_NAME, 'a', buffering=1) as visited_fh:
+ logging.debug(f'Opened {VISITED_FILE_NAME} to find already visited ids')
+ with open(LOG_FILE_NAME, 'w', buffering=1)as report_fh:
+ logging.debug(f'Opened {LOG_FILE_NAME} to write report to')
+ with mp.Pool(4) as pool:
+ completed_jobs \
+ = pool.imap_unordered(f_then_c, ids)
+
+ def done_job(job):
+ (res_dict, bad_results) = job
+ logging.debug(f"completed {res_dict['id']}")
+ visited_fh.write(f"{res_dict['id']}\n")
+ write_comparison_org(report_fh, (res_dict, list(bad_results)))
+ if done():
+ logging.info("done and existing")
+ exit(0)
+
+ [done_job(job) for job in completed_jobs]
+
+
+
+def protect(fn: Callable[[Any],BadResult] )-> Callable[[Any],BadResult]:
+ """Return function that will not throw"""
+ def protected(res_args:Dict) -> BadResult:
+ # noinspection PyBroadException
+ try:
+ return fn(res_args)
+ except Exception as ex:
+ return BadResult(res_args['id'], "name unknown", traceback.format_exc())
+ return protected
+
+
+def _serialize_obj(obj):
+ """JSON serializer for objects not serializable by default json code"""
+ return obj.__dict__
+
+def write_comparison(report_fh, result: Tuple[Dict, List[BadResult]]) -> None:
+ (config, bad_results) = result
+ logging.debug("writing report for %s", config['id'])
+ report_fh.write( json.dumps(config, sort_keys=True) + "\n")
+ if bad_results:
+ report_fh.write( json.dumps(bad_results, sort_keys=True, default=_serialize_obj) + "\n")
+ else:
+ report_fh.write("no bad results\n")
+
+def write_comparison_org(report_fh, result: Tuple[Dict, List[BadResult]]) -> None:
+ (config, bad_results) = result
+ logging.debug("writing report for %s", config['id'])
+ report_fh.write(f"* {config['id']} \n")
+
+ report_fh.write(f"** config for {config['id']}\n")
+ report_fh.write(json.dumps(config, sort_keys=True) + "\n")
+
+ report_fh.write("** Results\n")
+ if bad_results:
+ for result in bad_results:
+ report_fh.write(f"*** {result.comparison} \n")
+ #report_fh.write( json.dumps(result, sort_keys=True, default=_serialize_obj) + "\n")
+ report_fh.write(result.message + "\n")
+ report_fh.write(f"**** NG value: \n{result.ng}\n")
+ report_fh.write(f"**** Legacy value: \n{result.legacy}\n")
+ else:
+ report_fh.write("No bad results\n")
+
+
+def format_bad_result(bad: BadResult)->str:
+ rpt = f"** {bad.comparison}\n" \
+ f"{bad.message} "
+ if bad.similarity:
+ rpt = rpt + f"sim: {bad.similarity}\n"
+ else:
+ rpt = rpt + "\n"
+
+ if bad.legacy or bad.ng:
+ rpt = rpt + f"Legacy: '{bad.legacy}'\nNG: '{bad.ng}'\n"
+
+ return rpt
+
+
+#this is just to get around passing the config to the pool
+def dict_from_module(module):
+ context = {}
+ for setting in required_keys:
+ context[setting] = getattr(module, setting)
+ return context
+
+if __name__ == '__main__':
+ main()
diff --git a/tests/legacy_comparison/response_comparisons.py b/tests/legacy_comparison/response_comparisons.py
index 83143e2a8..e6d48e552 100644
--- a/tests/legacy_comparison/response_comparisons.py
+++ b/tests/legacy_comparison/response_comparisons.py
@@ -1,4 +1,4 @@
-from tests.legacy_comparison.comparison_types import res_arg_dict, BadResult
+from comparison_types import res_arg_dict, BadResult
def compare_status(res_arg: res_arg_dict) -> BadResult:
diff --git a/tests/legacy_comparison/text_comparisons.py b/tests/legacy_comparison/text_comparisons.py
index 50740cf2f..4dd019e18 100644
--- a/tests/legacy_comparison/text_comparisons.py
+++ b/tests/legacy_comparison/text_comparisons.py
@@ -1,5 +1,5 @@
-from tests.legacy_comparison.abstract_comparisons import lev_similarity
-from tests.legacy_comparison.comparison_types import text_arg_dict, BadResult
+from abstract_comparisons import lev_similarity
+from comparison_types import text_arg_dict, BadResult
def text_similarity(text_arg: text_arg_dict) -> BadResult:
diff --git a/tests/test_404.py b/tests/test_404.py
index 1cb9cb4a8..5babaeae5 100644
--- a/tests/test_404.py
+++ b/tests/test_404.py
@@ -1,10 +1,11 @@
import unittest
-import logging
+import logging
from app import app
-
-class It_Should_be_404_Test(unittest.TestCase):
-
+
+
+class Test_404(unittest.TestCase):
+
def setUp(self):
"""Disable logging to avoid messy output during testing"""
import logging
@@ -15,11 +16,10 @@ def setUp(self):
app.config['APPLICATION_ROOT'] = ''
self.app = app.test_client()
- def its_should_be_404_test(self):
+ def test_it_should_be_404(self):
rv = self.app.get('/abs?archive=foo&papernum=1234567')
self.assertEqual(rv.status_code, 404)
rv = self.app.get('/abs?0704.0600')
self.assertEqual(rv.status_code, 404,
'singleton case for new IDs not supported')
-
diff --git a/tests/test_archive.py b/tests/test_archive.py
new file mode 100644
index 000000000..b4b80b7e8
--- /dev/null
+++ b/tests/test_archive.py
@@ -0,0 +1,71 @@
+import unittest
+
+from app import app
+
+
+class BrowseTest(unittest.TestCase):
+ def setUp(self):
+ app.testing = True
+ app.config["APPLICATION_ROOT"] = ""
+ self.app = app.test_client()
+
+ def test_astroph_archive(self):
+ rv = self.app.get("/archive/astro-ph")
+ self.assertEqual(rv.status_code, 200)
+ self.assertIn('Expires', rv.headers, 'Should have expires header')
+
+ src = rv.data.decode("utf-8")
+ self.assertIn("Astrophysics", src)
+ self.assertIn("/year/astro-ph/92", src)
+ self.assertIn("/year/astro-ph/19", src)
+
+ self.assertIn(
+ "Astrophysics of Galaxies",
+ src,
+ "Subcategories of astro-ph should be on archive page",
+ )
+ self.assertIn(
+ "Earth and Planetary Astrophysics",
+ src,
+ "Subcategories of astro-ph should be on archive page",
+ )
+
+ def test_list(self):
+ rv = self.app.get("/archive/list")
+ self.assertEqual(rv.status_code, 200)
+ src = rv.data.decode("utf-8")
+
+ self.assertIn("Astrophysics", src)
+ self.assertIn("astro-ph", src)
+
+ self.assertIn("Materials Theory", src)
+ self.assertIn("mtrl-th", src)
+
+ rv = self.app.get("/archive/bogus-archive")
+ self.assertEqual(rv.status_code, 404)
+
+ def test_subsumed_archive(self):
+ rv = self.app.get("/archive/comp-lg")
+ self.assertEqual(rv.status_code, 404)
+ src = rv.data.decode("utf-8")
+
+ self.assertIn("Computer Science", src)
+ self.assertIn("cs.CL", src)
+
+ rv = self.app.get("/archive/acc-phys")
+ self.assertEqual(rv.status_code, 200)
+ src = rv.data.decode("utf-8")
+
+ self.assertIn("Accelerator Physics", src)
+ self.assertIn("physics.acc-ph", src)
+
+ def test_single_archive(self):
+ rv = self.app.get("/archive/hep-ph")
+ self.assertEqual(rv.status_code, 200)
+ src = rv.data.decode("utf-8")
+
+ self.assertIn("High Energy Physics", src)
+ self.assertNotIn("Categories within", src)
+
+
+
diff --git a/tests/test_author_affil.py b/tests/test_author_affil.py
deleted file mode 100644
index e240ad166..000000000
--- a/tests/test_author_affil.py
+++ /dev/null
@@ -1,291 +0,0 @@
-"""Tests for author and affiliation parsing."""
-from unittest import TestCase
-
-from browse.services.document.author_affil import parse_author_affil, split_authors
-from browse.services.document.metadata import AbsMetaSession
-from tests import path_of_for_test
-
-
-class TestAuthorAffiliationParsing(TestCase):
-
- def test_split_authors(self):
- self.assertListEqual(split_authors('Simeon Warner'),
- ['Simeon Warner'])
-
- self.assertListEqual(split_authors('The DELPHI Collaboration, J. Abdallah, et al'),
- ['The DELPHI Collaboration', ',', 'J. Abdallah', ',', 'et al'])
-
- self.assertListEqual(split_authors('BELLE Collaboration: A Person, Nother Person'),
- ['BELLE Collaboration', ':', 'A Person', ',', 'Nother Person'])
-
- self.assertListEqual(split_authors('Simeon Warner, Herbert Van de Sompel'),
- ['Simeon Warner', ',', 'Herbert Van de Sompel'])
-
- self.assertListEqual(
- split_authors('An Author'),
- ['An Author'],
- 'single author'
- )
-
- self.assertListEqual(
- split_authors(''),
- [],
- 'empty author'
- )
-
- self.assertListEqual(
- split_authors('An Author (affil)'),
- ['An Author', '(affil)'],
- 'single author with affil'
- )
- self.assertListEqual(
- split_authors('An Author (affil)'),
- ['An Author', '(affil)'],
- 'single author with affil'
- )
- self.assertListEqual(
- split_authors('An Author and Another P. H. J. Author (affil)'),
- ['An Author', ',', 'Another P. H. J. Author', '(affil)'],
- 'double author with affil'
- )
- self.assertListEqual(
- split_authors(
- 'John Von Neumann, Herbert Van de Sompel, Fred Bloggs, Jr, et al'),
- ['John Von Neumann', ',', 'Herbert Van de Sompel',
- ',', 'Fred Bloggs, Jr', ',', 'et al'],
- 'multiple with prefixes and suffixes'
- )
- self.assertListEqual(
- split_authors('sum won ( whatever affil data unmunged )'),
- ['sum won', '( whatever affil data unmunged )'],
- 'one author, two labs'
- )
- self.assertListEqual(
- split_authors('sum won(1,2)((1)lab a,(2)lab b)'),
- ['sum won', '(1,2)', '((1)lab a,(2)lab b)'],
- 'one author, two labs'
- )
-
- def test_parse_author_affil_basic(self):
- self.assertListEqual(parse_author_affil(
- 'Simeon Warner'), [['Warner', 'Simeon', '']])
-
- self.assertListEqual(
- parse_author_affil('Simeon Warner Jr'),
- [['Warner', 'Simeon', 'Jr']])
-
- self.assertListEqual(
- parse_author_affil('Simeon Warner Jr.'),
- [['Warner', 'Simeon', 'Jr.']])
-
- self.assertListEqual(
- parse_author_affil('Simeon Warner Sr'),
- [['Warner', 'Simeon', 'Sr']])
-
- self.assertListEqual(
- parse_author_affil('Simeon Warner Sr.'),
- [['Warner', 'Simeon', 'Sr.']])
-
- self.assertListEqual(
- parse_author_affil('SM Warner'),
- [['Warner', 'SM', '']])
-
- self.assertListEqual(
- parse_author_affil('SM. Warner'),
- [['Warner', 'SM.', '']])
-
- def test_parse_author_affil_basic2(self):
- self.assertListEqual(
- parse_author_affil('S.M. Warner'),
- [['Warner', 'S. M.', '']])
-
- self.assertListEqual(
- parse_author_affil('John Von Neumann'),
- [['Von Neumann', 'John', '']])
-
- self.assertListEqual(
- parse_author_affil('Herbert Van de Sompel'),
- [['Van de Sompel', 'Herbert', '']])
-
- self.assertListEqual(
- parse_author_affil('del Norte'),
- [['Norte', 'del', '']])
-
- self.assertListEqual(
- parse_author_affil('Fred del Norte'),
- [['del Norte', 'Fred', '']])
-
- self.assertListEqual(
- parse_author_affil('BELLE'),
- [['BELLE', '', '']])
-
- self.assertListEqual(
- parse_author_affil('BELLE Collaboration: A Person, Nother Person'),
- [
- ['BELLE Collaboration', '', ''],
- ['Person', 'A', ''],
- ['Person', 'Nother', '']
- ])
-
- self.assertListEqual(parse_author_affil('The DELPHI Collaboration, J. Abdallah, et al'),
- [['The DELPHI Collaboration', '', ''], ['Abdallah', 'J.', '']])
-
- self.assertListEqual(parse_author_affil('Ali Vaziri Astaneh, Federico Fuentes'),
- [['Vaziri Astaneh', 'Ali', ''],['Fuentes', 'Federico', '']])
-
- def test_parse_author_affil_with_affiliations(self):
- self.assertListEqual(
- parse_author_affil('sum won (lab a)'),
- [['won', 'sum', '', 'lab a']])
-
- self.assertListEqual(
- parse_author_affil('sum won (lab a; lab b)'),
- [['won', 'sum', '', 'lab a; lab b']])
-
- self.assertListEqual(
- parse_author_affil('sum won (lab a, lab b)'),
- [['won', 'sum', '', 'lab a, lab b']])
-
- self.assertListEqual(
- parse_author_affil('sum won (1,2) ( (1) lab a, (2) lab b)'),
- [['won', 'sum', '', 'lab a', 'lab b']])
-
- self.assertListEqual(
- parse_author_affil('sum won(1,2)((1)lab a,(2)lab b)'),
- [['won', 'sum', '', 'lab a', 'lab b']])
-
- self.assertListEqual(
- parse_author_affil('a.b.first, c.d.second (affil)'),
- [['first', 'a. b.', '', 'affil'], ['second', 'c. d.', '', 'affil']])
-
- self.assertListEqual(
- parse_author_affil('a.b.first (affil), c.d.second (affil)'),
- [['first', 'a. b.', '', 'affil'], ['second', 'c. d.', '', 'affil']])
-
- self.assertListEqual(
- parse_author_affil(
- 'a.b.first, c.d.second (1), e.f.third, g.h.forth (2,3) ((1) affil1, (2) affil2, (3) affil3)'
- ),
- [
- ['first', 'a. b.', '', 'affil1'],
- ['second', 'c. d.', '', 'affil1'],
- ['third', 'e. f.', '', 'affil2', 'affil3'],
- ['forth', 'g. h.', '', 'affil2', 'affil3']
- ])
-
- self.assertListEqual(
- parse_author_affil((
- "QUaD collaboration: S. Gupta (1), P. Ade (1), J. Bock (2,3), M. Bowden "
- "(1,4), M. L. Brown (5), G. Cahill (6), P. G. Castro (7,8), S. Church (4), T. "
- "Culverhouse (9), R. B. Friedman (9), K. Ganga (10), W. K. Gear (1), J. "
- "Hinderks (5,11), J. Kovac (3), A. E. Lange (4), E. Leitch (2,3), S. J. "
- "Melhuish (12), Y. Memari (7), J. A. Murphy (6), A. Orlando (1,3), C. "
- "O'Sullivan (6), L. Piccirillo (12), C. Pryke (9), N. Rajguru (1,13), B. "
- "Rusholme (4,14), R. Schwarz (9), A. N. Taylor (7), K. L. Thompson (4), A. H. "
- "Turner (1), E. Y. S. Wu (4), M. Zemcov (1,2,3) ((1) Cardiff University, (2) "
- "JPL, (3) Caltech, (4) Stanford University, (5) University of Cambridge, (6) "
- "National University of Ireland Maynooth, (7) University of Edinburgh, (8) "
- "Universidade Tecnica de Lisboa, (9) University of Chicago, (10) Laboratoire "
- "APC/CNRS, (11) NASA Goddard, (12) University of Manchester, (13) UCL, (14) "
- "IPAC) "
- )),
- [
- ["QUaD collaboration", "", ""],
- ["Gupta", "S.", "", "Cardiff University"],
- ["Ade", "P.", "", "Cardiff University"],
- ["Bock", "J.", "", "JPL", "Caltech"],
- ["Bowden", "M.", "", "Cardiff University", "Stanford University"],
- ["Brown", "M. L.", "", "University of Cambridge"],
- ["Cahill", "G.", "", "National University of Ireland Maynooth"],
- [
- "Castro", "P. G.", "",
- "University of Edinburgh",
- "Universidade Tecnica de Lisboa"
- ],
- ["Church", "S.", "", "Stanford University"],
- ["Culverhouse", "T.", "", "University of Chicago"],
- ["Friedman", "R. B.", "", "University of Chicago"],
- ["Ganga", "K.", "", "Laboratoire APC/CNRS"],
- ["Gear", "W. K.", "", "Cardiff University"],
- ["Hinderks", "J.", "", "University of Cambridge", "NASA Goddard"],
- ["Kovac", "J.", "", "Caltech"],
- ["Lange", "A. E.", "", "Stanford University"],
- ["Leitch", "E.", "", "JPL", "Caltech"],
- ["Melhuish", "S. J.", "", "University of Manchester"],
- ["Memari", "Y.", "", "University of Edinburgh"],
- ["Murphy", "J. A.", "", "National University of Ireland Maynooth"],
- ["Orlando", "A.", "", "Cardiff University", "Caltech"],
- ["O'Sullivan", "C.", "", "National University of Ireland Maynooth"],
- ["Piccirillo", "L.", "", "University of Manchester"],
- ["Pryke", "C.", "", "University of Chicago"],
- ["Rajguru", "N.", "", "Cardiff University", "UCL"],
- ["Rusholme", "B.", "", "Stanford University", "IPAC"],
- ["Schwarz", "R.", "", "University of Chicago"],
- ["Taylor", "A. N.", "", "University of Edinburgh"],
- ["Thompson", "K. L.", "", "Stanford University"],
- ["Turner", "A. H.", "", "Cardiff University"],
- ["Wu", "E. Y. S.", "", "Stanford University"],
- ["Zemcov", "M.", "", "Cardiff University", "JPL", "Caltech"]
- ],
- 'parse_author_affil (mind-blowing) 1/1'
- )
-
- # Problem case with 1110.4366
- self.assertListEqual(
- parse_author_affil(
- 'Matthew Everitt, Robert M. Heath and Viv Kendon'),
- [['Everitt', 'Matthew', ''],
- ['Heath', 'Robert M.', ''],
- ['Kendon', 'Viv', '']],
- 'parse_author_affil for 1110.4366'
- )
-
- # look like bugs, but aren't
- self.assertListEqual(
- parse_author_affil('sum won ((lab a), (lab b))'),
- [['won', 'sum', '']],
- 'parse_author_affil (bug imposter) 1/2'
- )
- self.assertListEqual(
- parse_author_affil('sum won ((lab a) (lab b))'),
- [['won', 'sum', '']],
- 'parse_author_affil (bug imposter) 2/2'
- )
-
- self.assertListEqual(
- parse_author_affil('Anatoly Zlotnik and Jr-Shin Li'),
- [['Zlotnik', 'Anatoly', ''],
- ['Li', 'Jr-Shin', '']],
- 'jr issue (Anatoly Zlotnik and Jr-Shin Li)'
- )
-
- # ====== Extra tests for arXiv::AuthorAffil ARXIVDEV-728 ======
-
- # [parse_author_affil]
- self.assertListEqual(
- parse_author_affil(''),
- [],
- 'parse_author_affil (empty)'
- )
- self.assertListEqual(
- parse_author_affil('Simeon Warner Jr'),
- [['Warner', 'Simeon', 'Jr']],
- 'parse_author_affil (basic) 2/12'
- )
- self.assertListEqual(
- parse_author_affil('BELLE Collaboration'),
- [['BELLE Collaboration', '', '']],
- 'parse_author_affil (lone "BELLE Collaboration") 2/3'
- )
-
- self.assertListEqual(
- parse_author_affil('BELLE Collaboration'),
- [['BELLE Collaboration', '', '']],
- 'parse_author_affil (lone "BELLE Collaboration") 2/3'
- )
-
- def test_collaboration_at_front(self):
- f1 = path_of_for_test('data/abs_files/ftp/arxiv/papers/0808/0808.4142.abs')
- meta = AbsMetaSession.parse_abs_file(filename=f1)
- paflst = parse_author_affil(meta.authors.raw)
- self.assertListEqual(paflst, [['D0 Collaboration', '', ''], ['Abazov', 'V.', '']])
diff --git a/tests/test_browse.py b/tests/test_browse.py
index 2c6cc8098..d14bc12cb 100644
--- a/tests/test_browse.py
+++ b/tests/test_browse.py
@@ -1,8 +1,9 @@
import unittest
from bs4 import BeautifulSoup
-
from tests.test_abs_parser import ABS_FILES
+
+from arxiv import taxonomy
from browse.services.document.metadata import AbsMetaSession
from browse.domain.license import ASSUMED_LICENSE_URI
@@ -18,6 +19,104 @@ def setUp(self):
app.config['APPLICATION_ROOT'] = ''
self.app = app.test_client()
+ def test_home(self):
+ """Test the home page."""
+ rv = self.app.get('/')
+ self.assertEqual(rv.status_code, 200)
+ html = BeautifulSoup(rv.data.decode('utf-8'), 'html.parser')
+
+ for group_key, group_value in taxonomy.definitions.GROUPS.items():
+ if group_key == 'grp_test':
+ continue
+ auths_elmt = html.find('h2', text=group_value['name'])
+ self.assertTrue(auths_elmt, f"{group_value['name']} in h2 element")
+ self.assertFalse(html.find('h2', text='Test'),
+ "'Test' group should not be shown on homepage")
+
+ def test_tb(self):
+ """Test the /tb/ page."""
+ rv = self.app.get('/tb/1901.99999')
+ self.assertEqual(rv.status_code, 404)
+
+ rv = self.app.get('/tb/')
+ self.assertEqual(rv.status_code, 404)
+
+ rv = self.app.get('/tb/foo')
+ self.assertEqual(rv.status_code, 404)
+
+ rv = self.app.get('/tb/0808.4142')
+ self.assertEqual(rv.status_code, 200)
+
+ html = BeautifulSoup(rv.data.decode('utf-8'), 'html.parser')
+ h2_elmt = html.find('h2')
+ h2_txt = h2_elmt.get_text()
+ self.assertTrue(h2_elmt, 'Should have element')
+ self.assertEquals(h2_txt, 'Trackbacks for 0808.4142')
+ tb_a_tags = html.find_all('a', 'mathjax', rel='external nofollow')
+ self.assertGreater(len(tb_a_tags), 1,
+ 'There should be more than one tag for trackbacks')
+ h1_elmt = html.find('div', id='abs')
+ h1_txt = h1_elmt.get_text()
+ self.assertTrue(h1_elmt, 'Should have
'))
+ )
+
+ s = f'something something {doi} endthing'
+ doiurl = urlize_dois(s)
+ self.assertRegex(
+ doiurl, r'10.1103/PhysRevA.99.013009 '
+ '10.1103/PhysRevZ.44.023009 '
+ '10.1103/PhysRevX.90.012309 '
+ '10.1103/BioRevX.44.123456'
+ )
+ )
+ )
+
+ txt = ''
+ self.assertEqual(
+ urlize_dois(f'{doi} {txt}'),
+ str(Markup(f'10.1103/PhysRevD.76.013009 '))
+ )
def test_arxiv_id_urls_basic(self):
- h = 'sosmooth.org' # Totally bogus setup for testing, at least url_for returns something
+ """Test basic urlize for arXiv identifiers."""
+ # a server name is needed for url_for to return something
+ h = 'arxiv.org'
app.config['SERVER_NAME'] = h
with app.app_context():
- assert_that(arxiv_id_urls(''), equal_to(''))
+ self.assertEqual(urlize('', ['arxiv_id']), '')
s = 'some text 134#%$$%&^^%*^&(()*_)_<>?:;[}}'
- assert_that(
- arxiv_id_urls(s), equal_to(escape(s)),
- 'filers should return marked up text, which means its escaped')
- assert_that(
- arxiv_id_urls('hep-th/9901001'),
- equal_to(
- f'hep-th/9901001',
- ))
- assert_that(
- arxiv_id_urls('hep-th/9901001 hep-th/9901002'),
- equal_to(
- f'hep-th/9901001 hep-th/9901002'
- ))
+ self.assertEqual(urlize(s), str(escape(s)),
+ 'filters should return escaped text')
+ self.assertEqual(
+ urlize('hep-th/9901001'),
+ f'hep-th/9901001',
+ )
+ self.assertEqual(
+ urlize('hep-th/9901001 hep-th/9901002'),
+ f'hep-th/9901001 hep-th/9901002'
+ )
def test_arxiv_id_urls_3(self):
+ """Test more complex cases of urlize for arXiv identifiers."""
h = 'sosmooth.org'
app.config['SERVER_NAME'] = h
with app.app_context():
- assert_that(
- arxiv_id_urls('hep-th/9901002'),
- equal_to(
- f'hep-th/9901002',
- ))
- assert_that(
- arxiv_id_urls('hep-th/9901002\n'),
- equal_to(
- f'hep-th/9901002\n',
- ),
+ self.assertEqual(
+ urlize('hep-th/9901002'),
+ f'hep-th/9901002',
)
- assert_that(
- arxiv_id_urls(
- 'arXiv:dg-ga/9401001 hep-th/9901001 hep-th/9901002'),
- equal_to(
- f'arXiv:dg-ga/9401001 hep-th/9901001 hep-th/9901002',
- ),
+ self.assertEqual(
+ urlize('hep-th/9901002\n'),
+ f'hep-th/9901002\n'
+ )
+ self.assertEqual(
+ urlize('arXiv:dg-ga/9401001 hep-th/9901001 hep-th/9901002'),
+ f'arXiv:dg-ga/9401001 hep-th/9901001 hep-th/9901002'
)
def test_arxiv_id_urls_punct(self):
+ """Test cases of of urlize for arXiv identifiers with punctuation."""
h = 'sosmooth.org'
app.config['SERVER_NAME'] = h
with app.app_context():
- assert_that(
- arxiv_id_urls('hep-th/9901002.'),
- equal_to(
- f'hep-th/9901002.',
- ), 'followed by period')
- assert_that(
- arxiv_id_urls('0702.0003.'),
- equal_to(
- f'0702.0003.', ),
+ self.assertEqual(
+ urlize('hep-th/9901002.'),
+ f'hep-th/9901002.',
+ 'followed by period')
+ self.assertEqual(
+ urlize('0702.0003.'),
+ f'0702.0003.',
'followed by period')
- assert_that(
- arxiv_id_urls('hep-th/9901001,hep-th/9901002'),
- equal_to(
- f'hep-th/9901001,hep-th/9901002'
- ), 'filter_urls_ids_escape (ID linking) 3/7')
- assert_that(
- arxiv_id_urls('0702.0003, something'),
- equal_to(
- f'0702.0003, something',
- ), 'followed by comma')
- assert_that(
- arxiv_id_urls('(0702.0003) something'),
- equal_to(
- f'(0702.0003) something',
- ), 'in parens')
+ self.assertEqual(
+ urlize('hep-th/9901001,hep-th/9901002'),
+ f'hep-th/9901001,hep-th/9901002',
+ 'filter_urls_ids_escape (ID linking) 3/7')
+ self.assertEqual(
+ urlize('0702.0003, something'),
+ f'0702.0003, something',
+ 'followed by comma')
+ self.assertEqual(
+ urlize('(0702.0003) something'),
+ f'(0702.0003) something',
+ 'in parens')
def test_arxiv_id_urls_more(self):
+ """Test urlize for arXiv identifiers that have mixed formatting."""
h = 'sosmooth.org'
app.config['SERVER_NAME'] = h
with app.app_context():
self.assertEqual(
- arxiv_id_urls('arXiv:dg-ga/9401001 hep-th/9901001 0704.0001'),
- f'arXiv:dg-ga/9401001 hep-th/9901001 0704.0001',
- 'filter_urls_ids_escape (ID linking) 5/7')
+ urlize('arXiv:dg-ga/9401001 hep-th/9901001 0704.0001'),
+ f'arXiv:dg-ga/9401001 hep-th/9901001 0704.0001',
+ 'urlize (ID linking) 5/7')
def test_arxiv_id_v(self):
+ """Test urlize for arXiv identifers with version affix."""
h = 'sosmooth.org'
app.config['SERVER_NAME'] = h
with app.app_context():
- assert_that(
- arxiv_id_urls(
- 'arXiv:dg-ga/9401001v12 hep-th/9901001v2 0704.0001v1'),
- equal_to(
- f'arXiv:dg-ga/9401001v12 hep-th/9901001v2 0704.0001v1'
- ), 'arxiv ids with version numbers')
+ self.assertEqual(
+ urlize('arXiv:dg-ga/9401001v12 hep-th/9901001v2 0704.0001v1'),
+ f'arXiv:dg-ga/9401001v12 hep-th/9901001v2 0704.0001v1',
+ 'arxiv ids with version numbers')
+ @unittest.skip("TODO: confirm actual desired behavior")
def test_vixra(self):
+ """Test urlize for identifiers prefixed by viXra."""
h = 'sosmooth.org'
app.config['SERVER_NAME'] = h
with app.app_context():
- assert_that(
- arxiv_id_urls('viXra:0704.0001 viXra:1003.0123'),
- equal_to('viXra:0704.0001 viXra:1003.0123'))
+ self.assertEqual(urlize('viXra:0704.0001 viXra:1003.0123'),
+ 'viXra:0704.0001 viXra:1003.0123')
# this is what was expected in legacy, but it doesn't seem right:
- # assert_that(
- # arxiv_id_urls('vixra:0704.0001'),
- # equal_to(f'vixra:0704.0001'))
+ # self.assertEqual(
+ # urlize('vixra:0704.0001'),
+ # f'vixra:0704.0001')
def test_arxiv_id_urls_escaping(self):
+ """Test proper escaping when urlize applied."""
h = 'sosmooth.org'
app.config['SERVER_NAME'] = h
with app.app_context():
ax_id = 'hep-th/9901002'
user_entered_txt = '
bla')) + """Test the abstract lf to br tag filter.""" + self.assertEqual(abstract_lf_to_br('blal\n bla'), 'blal\n
bla') - assert_that( - line_feed_to_br('\nblal\n bla'), equal_to('\nblal\n
bla')) + self.assertEqual(abstract_lf_to_br( + '\nblal\n bla'), '\nblal\n
bla') - assert_that( - line_feed_to_br('\n blal\n bla'), equal_to('\n blal\n
bla'), - 'need to not do
on first line') - assert_that( - line_feed_to_br('blal\n\nbla'), equal_to('blal\nbla'), - 'skip blank lines') + self.assertEqual(abstract_lf_to_br('\n blal\n bla'), + '\n blal\n
bla', + 'need to not do
on first line') + self.assertEqual(abstract_lf_to_br('blal\n\nbla'), 'blal\nbla', + 'skip blank lines') def test_line_break_jinja(self): + """Test the abstract lf to br tag filter with urlize on arXiv IDs.""" h = 'sosmooth.org' app.config['SERVER_NAME'] = h with app.app_context(): jenv = Environment(autoescape=True) - jenv.filters['arxiv_urlize'] = arxiv_urlize - jenv.filters['line_break'] = line_feed_to_br + jenv.filters['urlize'] = urlize + jenv.filters['line_break'] = abstract_lf_to_br - assert_that( + self.assertEqual( jenv.from_string( '{{" http://google.com something or \n' '\n' 'no double \\n' ' should have br\n' - 'hep-th/9901002 other"|line_break|arxiv_urlize}}' + 'hep-th/9901002 other"|line_break|urlize|safe}}' ).render(), - equal_to( - '<script>bad junk</script> ' - 'this http URL' - ' something or \n' - 'no double \n' - '
should have br\n' - 'hep-th/9901002 other' - ), + '<script>bad junk</script> ' + 'this http URL' + ' something or \n' + 'no double \n' + '
should have br\n' + 'hep-th/9901002 other', 'line_break and arxiv_id_urls should work together' ) - def test_tex_to_utf(self): + def test_tex2utf(self): + """Test the tex2utf filter.""" h = 'sosmooth.org' app.config['SERVER_NAME'] = h with app.app_context(): jenv = Environment(autoescape=True) - jenv.filters['arxiv_id_urls'] = arxiv_id_urls - jenv.filters['line_break'] = line_feed_to_br - jenv.filters['doi_urls'] = partial(doi_urls, lambda x: x) - jenv.filters['arxiv_urlize'] = arxiv_urlize - jenv.filters['tex_to_utf'] = tex_to_utf + jenv.filters['urlize'] = urlize + jenv.filters['line_break'] = abstract_lf_to_br + jenv.filters['tex2utf'] = f_tex2utf - assert_that( - jenv.from_string('{{""|tex_to_utf|arxiv_id_urls}}').render(), - equal_to('')) + self.assertEqual( + jenv.from_string('{{""|tex2utf|urlize|safe}}').render(), + '' + ) title = jenv.from_string( - '{{"Finite-Size and Finite-Temperature Effects in the Conformally Invariant O(N) Vector Model for 2 ARXIVNG-1227'
+ 'Finite-Size and Finite-Temperature Effects in the Conformally Invariant O(N) Vector Model for 2<d<4',
+ 'tex2utf and arxiv_id_urls should handle < and > ARXIVNG-1227'
)
- assert_that(tex_to_utf('Lu\\\'i'), equal_to('Luí'))
- assert_that(tex_to_utf(Markup('Lu\\\'i')), equal_to('Luí'))
- assert_that(tex_to_utf(Markup(escape('Lu\\\'i'))), equal_to('Luí'))
+ self.assertEqual(f_tex2utf('Lu\\\'i'), 'Luí')
+ self.assertEqual(f_tex2utf(Markup('Lu\\\'i')), 'Luí')
+ self.assertEqual(f_tex2utf(Markup(escape('Lu\\\'i'))), 'Luí')
def test_entity_to_utf(self):
+ """Test the entity to utf filter."""
h = 'sosmooth.org'
app.config['SERVER_NAME'] = h
with app.app_context():
jenv = Environment(autoescape=True)
- jenv.filters['arxiv_id_urls'] = arxiv_id_urls
- jenv.filters['line_break'] = line_feed_to_br
- jenv.filters['doi_urls'] = partial(doi_urls, lambda x: x)
- jenv.filters['arxiv_urlize'] = arxiv_urlize
- jenv.filters['tex_to_utf'] = tex_to_utf
jenv.filters['entity_to_utf'] = entity_to_utf
- assert_that(
+ self.assertEqual(
jenv.from_string('{{ "Martín"|entity_to_utf }}').render(),
- equal_to('Martín'), 'entity_to_utf should work')
- assert_that(
+ 'Martín', 'entity_to_utf should work')
+ self.assertEqual(
jenv.from_string(
'{{ ""|entity_to_utf }}').render(),
- equal_to('<Martín>'),
+ '<Martín>',
'entity_to_utf should work even with < or >')
def test_arxiv_urlize_no_email_links(self):
+ """Test to ensure email addresses are not turned into links."""
h = 'sosmooth.org'
app.config['SERVER_NAME'] = h
with app.app_context():
jenv = Environment(autoescape=True)
- jenv.filters['arxiv_urlize'] = arxiv_urlize
+ jenv.filters['urlize'] = urlize
- assert_that(
+ self.assertEqual(
jenv.from_string(
- '{{ "bob@example.com"|arxiv_urlize }}').render(),
- equal_to('bob@example.com'),
+ '{{ "bob@example.com"|urlize|safe }}').render(),
+ 'bob@example.com',
'arxiv_urlize should not turn emails into links')
- assert_that(
+ self.assertEqual(
jenv.from_string(
- '{{ ""|arxiv_urlize }}').render(),
- equal_to('<bob@example.com>'),
+ '{{ ""|urlize|safe }}').render(),
+ '<bob@example.com>',
'arxiv_urlize should work even with < or >')
def test_arxiv_urlize(self):
-
-
-
+ """Multiple basic urlize tests."""
h = 'sosmooth.org'
app.config['SERVER_NAME'] = h
with app.app_context():
-
- def do_arxiv_urlize(txt):
- return arxiv_urlize(txt)
-
self.assertEqual(
- do_arxiv_urlize('http://example.com/'),
- 'this http URL',
- 'do_arxiv_urlize (URL linking) 1/6')
+ urlize('http://example.com/'),
+ 'this http URL',
+ 'urlize (URL linking) 1/6')
self.assertEqual(
- do_arxiv_urlize('https://example.com/'),
- 'this https URL',
- 'do_arxiv_urlize (URL linking) 2/6')
+ urlize('https://example.com/'),
+ 'this https URL',
+ 'urlize (URL linking) 2/6')
self.assertEqual(
- do_arxiv_urlize('ftp://example.com/'),
- 'this ftp URL',
- 'do_arxiv_urlize (URL linking) 3/6')
+ urlize('ftp://example.com/'),
+ 'this ftp URL',
+ 'urlize (URL linking) 3/6')
self.assertEqual(
- do_arxiv_urlize('http://example.com/.hep-th/9901001'),
- 'this http URL',
- 'do_arxiv_urlize (URL linking) 4/6')
+ urlize('http://example.com/.hep-th/9901001'),
+ 'this http URL',
+ 'urlize (URL linking) 4/6')
self.assertEqual(
- do_arxiv_urlize(
+ urlize(
'http://projecteuclid.org/euclid.bj/1151525136'
),
- 'this http URL',
- 'do_arxiv_urlize (URL linking) 6/6')
- assert_that(
- do_arxiv_urlize(' Correction to Bernoulli (2006), 12, 551--570 http://projecteuclid.org/euclid.bj/1151525136'),
- equal_to(Markup(' Correction to Bernoulli (2006), 12, 551--570 this http URL')),
- 'do_arxiv_urlize (URL linking) 6/6')
+ 'this http URL',
+ 'urlize (URL linking) 6/6')
+ self.assertEqual(
+ urlize(
+ ' Correction to Bernoulli (2006), 12, 551--570 http://projecteuclid.org/euclid.bj/1151525136'),
+ Markup(' Correction to Bernoulli (2006), 12, 551--570 this http URL'),
+ 'urlize (URL linking) 6/6')
# shouldn't match
self.assertEqual(
- do_arxiv_urlize('2448446.4710(5)'), '2448446.4710(5)',
- 'do_arxiv_urlize (should not match) 1/9')
+ urlize('2448446.4710(5)'), '2448446.4710(5)',
+ 'urlize (should not match) 1/9')
self.assertEqual(
- do_arxiv_urlize('HJD=2450274.4156+/-0.0009'),
+ urlize('HJD=2450274.4156+/-0.0009'),
'HJD=2450274.4156+/-0.0009',
- 'do_arxiv_urlize (should not match) 2/9')
+ 'urlize (should not match) 2/9')
self.assertEqual(
- do_arxiv_urlize('T_min[HJD]=49238.83662(14)+0.146352739(11)E.'),
+ urlize('T_min[HJD]=49238.83662(14)+0.146352739(11)E.'),
'T_min[HJD]=49238.83662(14)+0.146352739(11)E.',
- 'do_arxiv_urlize (should not match) 3/9')
+ 'urlize (should not match) 3/9')
self.assertEqual(
- do_arxiv_urlize('Pspin=1008.3408s'), 'Pspin=1008.3408s',
- 'do_arxiv_urlize (should not match) 4/9')
+ urlize('Pspin=1008.3408s'), 'Pspin=1008.3408s',
+ 'urlize (should not match) 4/9')
self.assertEqual(
- do_arxiv_urlize('2453527.87455^{+0.00085}_{-0.00091}'),
+ urlize('2453527.87455^{+0.00085}_{-0.00091}'),
'2453527.87455^{+0.00085}_{-0.00091}',
- 'do_arxiv_urlize (should not match) 5/9')
- self.assertEqual(
- do_arxiv_urlize('2451435.4353'), '2451435.4353',
- 'do_arxiv_urlize (should not match) 6/9')
- assert_that(
- do_arxiv_urlize('cond-mat/97063007'),
- equal_to('cond-mat/97063007'),
- 'do_arxiv_urlize (should match) 7/9')
-
- assert_that(
- do_arxiv_urlize('[http://onion.com/something-funny-about-arxiv-1234]'),
- equal_to('[this http URL]'))
-
- assert_that(
- do_arxiv_urlize('[http://onion.com/?q=something-funny-about-arxiv.1234]'),
- equal_to('[this http URL]'))
-
- assert_that(
- do_arxiv_urlize('http://onion.com/?q=something funny'),
- equal_to('this http URL funny'),
+ 'urlize (should not match) 5/9')
+ self.assertEqual(
+ urlize('2451435.4353'), '2451435.4353',
+ 'urlize (should not match) 6/9')
+ self.assertEqual(
+ urlize('cond-mat/97063007'),
+ 'cond-mat/97063007',
+ 'urlize (should match) 7/9')
+
+ self.assertEqual(
+ urlize('[http://onion.com/something-funny-about-arxiv-1234]'),
+ '[this http URL]')
+
+ self.assertEqual(
+ urlize('[http://onion.com/?q=something-funny-about-arxiv.1234]'),
+ '[this http URL]')
+
+ self.assertEqual(
+ urlize('http://onion.com/?q=something funny'),
+ 'this http URL funny',
'Spaces CANNOT be expected to be part of URLs')
-
- assert_that(
- do_arxiv_urlize('"http://onion.com/something-funny-about-arxiv-1234"'),
- equal_to(Markup('"this http URL"')),
- 'Should handle URL surrounded by double quotes')
+ self.assertEqual(
+ urlize('"http://onion.com/something-funny-about-arxiv-1234"'),
+ '"this http URL"',
+ 'Should handle URL surrounded by double quotes')
diff --git a/tests/test_id_patterns.py b/tests/test_id_patterns.py
deleted file mode 100644
index b566c95d5..000000000
--- a/tests/test_id_patterns.py
+++ /dev/null
@@ -1,253 +0,0 @@
-import unittest
-from hamcrest import *
-import re
-
-from jinja2 import Markup, escape
-from browse.util.id_patterns import _find_match, Matchable, dois_ids_and_urls, \
- _transform_token, do_dois_id_urls_to_tags
-
-
-class Id_Patterns_Test(unittest.TestCase):
-
- def test_basic(self):
- _find_match([], 'test')
-
- m = _find_match( [Matchable([], re.compile(r'test'))], 'test')
- assert_that(m, is_not(None))
-
- m0 = Matchable([], re.compile(r'test'))
- m1 = Matchable([], re.compile(r'tests'))
-
- m = _find_match( [m0, m1], 'test')
- assert_that(m, is_not(None))
- assert_that(m[1], equal_to(m0))
-
- m = _find_match([m0, m1], 'tests')
- assert_that(m, is_not(None))
- assert_that(m[1], equal_to(m0))
-
- m = _find_match([m1, m0], 'tests')
- assert_that(m, is_not(None))
- assert_that(m[1], equal_to(m1))
-
- def test_arxiv_ids(self):
- def find_match(txt):
- return _find_match(dois_ids_and_urls, txt)
-
- assert_that(find_match('math/9901123'), is_not(None))
- assert_that(find_match('hep-ex/9901123'), is_not(None))
- assert_that(find_match('gr-qc/9901123'), is_not(None))
-
- assert_that(find_match('1202.1234'), is_not(None))
- assert_that(find_match('1202.1234v1'), is_not(None))
- assert_that(find_match('1203.12345'), is_not(None))
- assert_that(find_match('1203.12345v1'), is_not(None))
- assert_that(find_match('1203.12345v12'), is_not(None))
-
- # slightly odd but seen in comments
- assert_that(find_match('hep-ph/1203.12345v12'), is_not(None))
-
- def test_find_match(self):
- def find_match(txt):
- return _find_match(dois_ids_and_urls, txt)
-
- assert_that(find_match('junk'), equal_to(None))
- assert_that(find_match(''), equal_to(None))
- assert_that(find_match(' '), equal_to(None))
-
- assert_that(find_match('doi:10.1002/0470841559.ch1'), is_not(None))
- assert_that(find_match('doi:10.1038/nphys1170'), is_not(None))
-
- assert_that(find_match('http://arxiv.org'), is_not(None))
- assert_that(find_match('http://arxiv.org?something=1'), is_not(None))
- assert_that(find_match(
- 'http://arxiv.org?something=1&anohter=2'), is_not(None))
- assert_that(find_match('"http://arxiv.org"'), is_not(None))
-
- def test_transform_token(self):
- # def doi_id_url_transform_token(tkn,fn):
- # return doi_id_url_transform_token(fn, tkn)
-
- assert_that(
- do_dois_id_urls_to_tags( None, None, ''),
- equal_to(''))
-
- assert_that(
- do_dois_id_urls_to_tags( None, None,
- 'it is fine, chapter 234 see'),
- equal_to(Markup(escape('it is fine, chapter 234 see'))))
-
- assert_that(
- do_dois_id_urls_to_tags( None, None, 'http://arxiv.org'),
- equal_to('this http URL'))
-
- assert_that(
- do_dois_id_urls_to_tags( None, None,
- 'Stuff in the front http://arxiv.org other stuff'),
- equal_to('Stuff in the front this http URL other stuff'))
-
- assert_that(
- do_dois_id_urls_to_tags( None, None, '.http://arxiv.org.'),
- equal_to('.this http URL.'))
-
- assert_that(
- do_dois_id_urls_to_tags( None, None, '"http://arxiv.org"'),
- equal_to(Markup('"this http URL"')))
-
- def test_urlize(self):
- def do_arxiv_urlize(txt):
- return do_dois_id_urls_to_tags(None,None, txt)
-
- assert_that(
- do_arxiv_urlize('http://example.com/'),
- equal_to('this http URL'),
- 'do_arxiv_urlize (URL linking) 1/6')
- assert_that(
- do_arxiv_urlize('https://example.com/'),
- equal_to('this https URL'),
- 'do_arxiv_urlize (URL linking) 2/6')
- assert_that(
- do_arxiv_urlize('ftp://example.com/'),
- equal_to('this ftp URL'),
- 'do_arxiv_urlize (URL linking) 3/6')
-
-
- assert_that(
- do_arxiv_urlize(
- 'http://projecteuclid.org/euclid.bj/1151525136'
- ),
- equal_to('this http URL'),
- 'do_arxiv_urlize (URL linking) 6/6')
- # assert_that(
- # do_arxiv_urlize(
- # ' Correction to Bernoulli (2006), 12, 551--570 http://projecteuclid.org/euclid.bj/1151525136'),
- # equal_to(
- # ' Correction to Bernoulli (2006), 12, 551--570 this http URL'),
- # 'do_arxiv_urlize (URL linking) 6/6')
- # shouldn't match
- assert_that(
- do_arxiv_urlize('2448446.4710(5)'), '2448446.4710(5)',
- equal_to('do_arxiv_urlize (should not match) 1/9'))
- self.assertEqual(
- do_arxiv_urlize('HJD=2450274.4156+/-0.0009'),
- 'HJD=2450274.4156+/-0.0009',
- 'do_arxiv_urlize (should not match) 2/9')
- assert_that(
- do_arxiv_urlize(
- 'T_min[HJD]=49238.83662(14)+0.146352739(11)E.'),
- equal_to('T_min[HJD]=49238.83662(14)+0.146352739(11)E.'),
- 'do_arxiv_urlize (should not match) 3/9')
- assert_that(
- do_arxiv_urlize('Pspin=1008.3408s'), 'Pspin=1008.3408s',
- equal_to('do_arxiv_urlize (should not match) 4/9'))
- assert_that(
- do_arxiv_urlize('2453527.87455^{+0.00085}_{-0.00091}'),
- equal_to('2453527.87455^{+0.00085}_{-0.00091}'),
- 'do_arxiv_urlize (should not match) 5/9')
- assert_that(
- do_arxiv_urlize('2451435.4353'), equal_to('2451435.4353'),
- 'do_arxiv_urlize (should not match) 6/9')
-
-
- assert_that(
- do_arxiv_urlize('cond-mat/97063007'),
- equal_to(
- 'cond-mat/97063007'),
- 'do_arxiv_urlize (should match) 7/9')
-
- assert_that(
- do_arxiv_urlize(
- '[http://onion.com/something-funny-about-arxiv-1234]'),
- equal_to('[this http URL]'))
-
- assert_that(
- do_arxiv_urlize(
- '[http://onion.com/?q=something-funny-about-arxiv.1234]'),
- equal_to('[this http URL]'))
-
- assert_that(
- do_arxiv_urlize('http://onion.com/?q=something funny'),
- equal_to(
- 'this http URL funny'),
- 'Spaces CANNOT be expected to be part of URLs')
-
- assert_that(
- do_arxiv_urlize(
- '"http://onion.com/something-funny-about-arxiv-1234"'),
- equal_to(
- Markup('"this http URL"')),
- 'Should handle URL surrounded by double quotes')
-
- assert_that(
- do_arxiv_urlize('< http://example.com/1<2 ><'),
- equal_to('< this http URL<2 ><'),
- 'do_arxiv_urlize (URL linking) 5/6')
-
- assert_that(
- do_arxiv_urlize('Accepted for publication in A&A. The data will be available via CDS, and can be found "http://atlasgal.mpifr-bonn.mpg.de/cgi-bin/ATLASGAL_FILAMENTS.cgi"'),
- equal_to('Accepted for publication in A&A. The data will be available via CDS, and can be found "this http URL"')
- )
-
- assert_that(
- do_arxiv_urlize('see http://www.tandfonline.com/doi/abs/doi:10.1080/15980316.2013.860928?journalCode=tjid20'),
- equal_to('see this http URL')
- )
-
- assert_that(
- do_arxiv_urlize('http://authors.elsevier.com/a/1TcSd,Ig45ZtO'),
- equal_to('this http URL'))
-
- def category_id_test(self):
- def do_arxiv_urlize(txt):
- return do_dois_id_urls_to_tags(None,None, txt)
-
- assert_that(
- do_arxiv_urlize('version of arXiv.math.GR/0512484 (2011).'),
- equal_to('version of arXiv.math.GR/0512484 (2011).'))
-
- def hosts_tests(self):
- def do_arxiv_urlize(txt):
- return do_dois_id_urls_to_tags(None,None, txt)
-
- assert_that(do_arxiv_urlize('can be downloaded from http://rwcc.bao.ac.cn:8001/swap/NLFFF_DBIE_code/HeHan_NLFFF_JGR.pdf'),
- equal_to("can be downloaded from this http URL"),
- "Should deal with ports correctly")
-
-
- assert_that(do_arxiv_urlize("images is at http://85.20.11.14/hosting/punsly/APJLetter4.2.07/"),
- equal_to('images is at this http URL'),
- "should deal with numeric IP correctly")
-
- def urls_with_plus(self):
- def do_arxiv_urlize(txt):
- return do_dois_id_urls_to_tags(lambda x: x, lambda x:x, txt)
-
- assert_that(do_arxiv_urlize('http://www.fkf.mpg.de/andersen/docs/pub/abstract2004+/pavarini_02.pdf'),
- equal_to("this http URL"),
- "Should deal with plus in URL correctly")
-
- def anchors_with_slash(self):
- def do_arxiv_urlize(txt):
- return do_dois_id_urls_to_tags(lambda x: x, lambda x:x, txt)
-
- assert_that(do_arxiv_urlize('https://dms.sztaki.hu/ecml-pkkd-2016/#/app/privateleaderboard'),
- equal_to(Markup("this https URL")),
- "Should deal with slash in URL anchor correctly")
-
- def ftp_test(self):
- cmt = "7 Pages; ftp://ftp%40micrognu%2Ecom:anon%40anon@ftp.micrognu.com/pnenp/conclusion.pdf"
- def do_arxiv_urlize(txt):
- return do_dois_id_urls_to_tags(lambda x: x, lambda x:x, txt)
-
- assert_that(do_arxiv_urlize(cmt),
- equal_to(Markup('7 Pages; this ftp URL')))
-
-
- def arxiv_prefix_test(self):
-
- def do_arxiv_urlize(txt):
- return do_dois_id_urls_to_tags(lambda x: x, lambda x:x, txt)
-
- cmt = "see arxiv:1201.12345"
- assert_that(do_arxiv_urlize(cmt),
- equal_to(Markup('see arXiv:1201.12345')))
diff --git a/tests/test_list_page.py b/tests/test_list_page.py
new file mode 100644
index 000000000..8ecee2f9c
--- /dev/null
+++ b/tests/test_list_page.py
@@ -0,0 +1,485 @@
+import unittest
+import re
+from hamcrest import *
+from unittest.mock import MagicMock
+
+from bs4 import BeautifulSoup
+
+from tests.test_abs_parser import ABS_FILES
+from browse.services.document.metadata import AbsMetaSession
+from browse.domain.license import ASSUMED_LICENSE_URI
+from browse.services.listing.fake_listings import FakeListingFilesService
+from browse.services.listing import ListingService, get_listing_service
+import os
+
+from app import app
+
+
+class ListPageTest(unittest.TestCase):
+
+ def setUp(self):
+ app.testing = True
+ app.config['APPLICATION_ROOT'] = ''
+ self.app = app
+ self.client = app.test_client()
+
+ def test_basic_lists(self):
+ rv = self.client.get('/list/hep-ph/0901')
+ self.assertEqual(rv.status_code, 200)
+ self.assertNotEqual(rv.headers.get('Expires', None), None)
+
+ rv = self.client.get('/list/hep-ph/09')
+ self.assertEqual(rv.status_code, 200)
+ self.assertNotEqual(rv.headers.get('Expires', None), None)
+
+ rv = self.client.get('/list/hep-ph/new')
+ self.assertEqual(rv.status_code, 200)
+ self.assertNotEqual(rv.headers.get('Expires', None), None)
+
+ rv = self.client.get('/list/hep-ph/current')
+ self.assertEqual(rv.status_code, 200)
+ self.assertNotEqual(rv.headers.get('Expires', None), None)
+
+ rv = self.client.get('/list/hep-ph/pastweek')
+ self.assertEqual(rv.status_code, 200)
+ self.assertNotEqual(rv.headers.get('Expires', None), None)
+
+ rv = self.client.get('/list/hep-ph/recent')
+ self.assertEqual(rv.status_code, 200)
+ self.assertNotEqual(rv.headers.get('Expires', None), None)
+
+ rv = self.client.get('/list/hep-ph/0901?skip=925&show=25')
+ self.assertEqual(rv.status_code, 200)
+ self.assertNotEqual(rv.headers.get('Expires', None), None)
+
+ rv = self.client.get('/list/astro-ph/04')
+ self.assertEqual(rv.status_code, 200)
+ self.assertNotEqual(rv.headers.get('Expires', None), None)
+
+ rv = self.client.get('/list/math/92')
+ self.assertEqual(rv.status_code, 200)
+ self.assertNotEqual(rv.headers.get('Expires', None), None)
+
+ rv = self.client.get('/list/math/9201')
+ self.assertEqual(rv.status_code, 200)
+
+ rv = self.client.get('/list/math/0101')
+ self.assertEqual(rv.status_code, 200)
+
+ rv = self.client.get('/list/math/0102')
+ self.assertEqual(rv.status_code, 200)
+
+ rv = self.client.get('/list/math/0103')
+ self.assertEqual(rv.status_code, 200)
+
+ rv = self.client.get('/list/math/0104')
+ self.assertEqual(rv.status_code, 200)
+
+ rv = self.client.get('/list/math/0105')
+ self.assertEqual(rv.status_code, 200)
+
+ rv = self.client.get('/list/math/0106')
+ self.assertEqual(rv.status_code, 200)
+
+ rv = self.client.get('/list/math/0107')
+ self.assertEqual(rv.status_code, 200)
+
+ rv = self.client.get('/list/math/0108')
+ self.assertEqual(rv.status_code, 200)
+
+ rv = self.client.get('/list/math/0109')
+ self.assertEqual(rv.status_code, 200)
+
+ rv = self.client.get('/list/math/0110')
+ self.assertEqual(rv.status_code, 200)
+
+ rv = self.client.get('/list/math/0111')
+ self.assertEqual(rv.status_code, 200)
+
+ rv = self.client.get('/list/math/0112')
+ self.assertEqual(rv.status_code, 200)
+
+ rv = self.client.get('/list/math/01')
+ self.assertEqual(rv.status_code, 200)
+
+ rv = self.client.get('/list/math/18')
+ self.assertEqual(rv.status_code, 200)
+
+ rv = self.client.get('/list/math/20') # year 2020
+ self.assertEqual(rv.status_code, 200)
+
+ rv = self.client.get('/list/math/30') # year 2030
+ self.assertEqual(rv.status_code, 200)
+
+ rv = self.client.get('/list/math/200101')
+ self.assertEqual(rv.status_code, 200)
+
+ def test_listing_authors(self):
+ rv = self.client.get('/list/hep-ph/0901')
+ self.assertEqual(rv.status_code, 200)
+ au = b'Eqab M. Rabei'
+ assert au in rv.data, f'Simple check for author {au} in response.'
+
+ html = BeautifulSoup(rv.data.decode('utf-8'), 'html.parser')
+
+ auDivs = html.find_all('div', 'list-authors')
+ assert_that(auDivs, not_none())
+ assert_that(len(auDivs), greater_than(
+ 5), 'Should have some .list-author divs')
+
+ first_aus = auDivs[0].find_all('a')
+ assert_that(first_aus, has_length(4),
+ 'expect 4 tags for first artcile "Fractional WKB Approximation"')
+
+ assert_that(first_aus[0].get_text(), equal_to('Eqab M. Rabei'))
+ assert_that(first_aus[1].get_text(),
+ equal_to('Ibrahim M. A. Altarazi'))
+ assert_that(first_aus[2].get_text(), equal_to('Sami I. Muslih'))
+ assert_that(first_aus[3].get_text(), equal_to('Dumitru Baleanu'))
+
+ assert_that(auDivs[0].get_text(), is_not(contains_string(' ,')),
+ 'Should not have a comma with a space in front of it')
+
+ def test_paging_first(self):
+ rv = self.client.get('/list/hep-ph/0901')
+ self.assertEqual(rv.status_code, 200)
+
+ rvdata = rv.data.decode('utf-8')
+ html = BeautifulSoup(rvdata, 'html.parser')
+
+ paging = html.find(id='dlpage').find_all('div')[0]
+ assert_that(paging, not_none())
+ tgs = paging.find_all(['span', 'a'])
+ assert_that(tgs, not_none())
+ assert_that(len(tgs), 6)
+
+ assert_that(tgs[0].name, equal_to('span'))
+ assert_that(tgs[0].get_text(), equal_to('1-25'))
+
+ assert_that(tgs[1].name, equal_to('a'))
+ assert_that(tgs[1].get_text(), equal_to('26-50'))
+
+ assert_that(tgs[2].name, equal_to('a'))
+ assert_that(tgs[2].get_text(), equal_to('51-75'))
+
+ assert_that(tgs[3].name, equal_to('a'))
+ assert_that(tgs[3].get_text(), equal_to('76-100'))
+
+ assert_that(tgs[4].name, equal_to('span'))
+ assert_that(tgs[4].get_text(), equal_to('...'))
+
+ assert_that(tgs[5].name, equal_to('a'))
+ assert_that(tgs[5].get_text(), equal_to('1001-1001'))
+
+ # find the first article index tag
+ first_index_atag = html.find(id='articles').find_all(
+ 'dt')[0].find('a', string=re.compile(r'\[\d*\]'))
+ assert_that(first_index_atag, not_none())
+ assert_that(first_index_atag['name'], equal_to('item1'))
+ assert_that(first_index_atag.string, equal_to('[1]'))
+
+ def test_paging_second(self):
+ rv = self.client.get('/list/hep-ph/0901?skip=25&show=25')
+ self.assertEqual(rv.status_code, 200)
+
+ rvdata = rv.data.decode('utf-8')
+ html = BeautifulSoup(rvdata, 'html.parser')
+
+ paging = html.find(id='dlpage').find_all('div')[0]
+ assert_that(paging, not_none())
+ tgs = paging.find_all(['span', 'a'])
+ assert_that(tgs, not_none())
+ assert_that(len(tgs), 7)
+
+ assert_that(tgs[0].name, equal_to('a'))
+ assert_that(tgs[0].get_text(), equal_to('1-25'))
+
+ assert_that(tgs[1].name, equal_to('span'))
+ assert_that(tgs[1].get_text(), equal_to('26-50'))
+
+ assert_that(tgs[2].name, equal_to('a'))
+ assert_that(tgs[2].get_text(), equal_to('51-75'))
+
+ assert_that(tgs[3].name, equal_to('a'))
+ assert_that(tgs[3].get_text(), equal_to('76-100'))
+
+ assert_that(tgs[4].name, equal_to('a'))
+ assert_that(tgs[4].get_text(), equal_to('101-125'))
+
+ assert_that(tgs[5].name, equal_to('span'))
+ assert_that(tgs[5].get_text(), equal_to('...'))
+
+ assert_that(tgs[6].name, equal_to('a'))
+ assert_that(tgs[6].get_text(), equal_to('1001-1001'))
+
+ # find the first article index tag
+ first_index_atag = html.find(id='articles').find_all(
+ 'dt')[0].find('a', string=re.compile(r'\[\d*\]'))
+ assert_that(first_index_atag, not_none())
+ assert_that(first_index_atag['name'], is_not(
+ 'item1'), 'first item index should not be 1')
+ assert_that(first_index_atag.string, equal_to('[26]'))
+
+ def test_paging_middle(self):
+ rv = self.client.get('/list/hep-ph/0901?skip=175&show=25')
+ self.assertEqual(rv.status_code, 200)
+
+ rvdata = rv.data.decode('utf-8')
+ html = BeautifulSoup(rvdata, 'html.parser')
+
+ paging = html.find(id='dlpage').find_all('div')[0]
+ assert_that(paging, not_none())
+ tgs = paging.find_all(['span', 'a'])
+ assert_that(tgs, not_none())
+ assert_that(len(tgs), 7)
+
+ assert_that(tgs[0].name, equal_to('a'))
+ assert_that(tgs[0].get_text(), equal_to('1-25'))
+
+ assert_that(tgs[1].name, equal_to('span'))
+ assert_that(tgs[1].get_text(), equal_to('...'))
+
+ assert_that(tgs[2].name, equal_to('a'))
+ assert_that(tgs[2].get_text(), equal_to('101-125'))
+
+ assert_that(tgs[3].name, equal_to('a'))
+ assert_that(tgs[3].get_text(), equal_to('126-150'))
+
+ assert_that(tgs[4].name, equal_to('a'))
+ assert_that(tgs[4].get_text(), equal_to('151-175'))
+
+ assert_that(tgs[5].name, equal_to('span'))
+ assert_that(tgs[5].get_text(), equal_to('176-200'))
+
+ assert_that(tgs[6].name, equal_to('a'))
+ assert_that(tgs[6].get_text(), equal_to('201-225'))
+
+ assert_that(tgs[7].name, equal_to('a'))
+ assert_that(tgs[7].get_text(), equal_to('226-250'))
+
+ assert_that(tgs[8].name, equal_to('a'))
+ assert_that(tgs[8].get_text(), equal_to('251-275'))
+
+ assert_that(tgs[9].name, equal_to('span'))
+ assert_that(tgs[9].get_text(), equal_to('...'))
+
+ assert_that(tgs[10].name, equal_to('a'))
+ assert_that(tgs[10].get_text(), equal_to('1001-1001'))
+
+ # find the first article index tag
+ first_index_atag = html.find(id='articles').find_all(
+ 'dt')[0].find('a', string=re.compile(r'\[\d*\]'))
+ assert_that(first_index_atag, not_none())
+ assert_that(first_index_atag['name'], is_not(
+ 'item1'), 'first item index should not be 1')
+ assert_that(first_index_atag.string, equal_to('[176]'))
+
+ def test_paging_last(self):
+ rv = self.client.get('/list/hep-ph/0901?skip=1000&show=25')
+ self.assertEqual(rv.status_code, 200)
+
+ rvdata = rv.data.decode('utf-8')
+ html = BeautifulSoup(rvdata, 'html.parser')
+
+ paging = html.find(id='dlpage').find_all('div')[0]
+ assert_that(paging, not_none())
+ tgs = paging.find_all(['span', 'a'])
+ assert_that(tgs, not_none())
+ assert_that(len(tgs), 7)
+
+ assert_that(tgs[0].name, equal_to('a'))
+ assert_that(tgs[0].get_text(), equal_to('1-25'))
+
+ assert_that(tgs[1].name, equal_to('span'))
+ assert_that(tgs[1].get_text(), equal_to('...'))
+
+ assert_that(tgs[2].name, equal_to('a'))
+ assert_that(tgs[2].get_text(), equal_to('926-950'))
+
+ assert_that(tgs[3].name, equal_to('a'))
+ assert_that(tgs[3].get_text(), equal_to('951-975'))
+
+ assert_that(tgs[4].name, equal_to('a'))
+ assert_that(tgs[4].get_text(), equal_to('976-1000'))
+
+ assert_that(tgs[5].name, equal_to('span'))
+ assert_that(tgs[5].get_text(), equal_to('1001-1001'))
+
+ # find the first article index tag
+ first_index_atag = html.find(id='articles').find_all(
+ 'dt')[0].find('a', string=re.compile(r'\[\d*\]'))
+ assert_that(first_index_atag, not_none())
+ assert_that(first_index_atag['name'], is_not(
+ 'item1'), 'first item index should not be 1')
+ assert_that(first_index_atag.string, equal_to('[1001]'))
+
+ def test_paging_penultimate(self):
+ rv = self.client.get('/list/hep-ph/0901?skip=975&show=25')
+ self.assertEqual(rv.status_code, 200)
+
+ rvdata = rv.data.decode('utf-8')
+ html = BeautifulSoup(rvdata, 'html.parser')
+
+ paging = html.find(id='dlpage').find_all('div')[0]
+ assert_that(paging, not_none())
+ tgs = paging.find_all(['span', 'a'])
+ assert_that(tgs, not_none())
+ assert_that(len(tgs), 7)
+
+ assert_that(tgs[0].name, equal_to('a'))
+ assert_that(tgs[0].get_text(), equal_to('1-25'))
+
+ assert_that(tgs[1].name, equal_to('span'))
+ assert_that(tgs[1].get_text(), equal_to('...'))
+
+ assert_that(tgs[2].name, equal_to('a'))
+ assert_that(tgs[2].get_text(), equal_to('901-925'))
+
+ assert_that(tgs[3].name, equal_to('a'))
+ assert_that(tgs[3].get_text(), equal_to('926-950'))
+
+ assert_that(tgs[4].name, equal_to('a'))
+ assert_that(tgs[4].get_text(), equal_to('951-975'))
+
+ assert_that(tgs[5].name, equal_to('span'))
+ assert_that(tgs[5].get_text(), equal_to('976-1000'))
+
+ assert_that(tgs[6].name, equal_to('a'))
+ assert_that(tgs[6].get_text(), equal_to('1001-1001'))
+
+ # find the first article index tag
+ first_index_atag = html.find(id='articles').find_all(
+ 'dt')[0].find('a', string=re.compile(r'\[\d*\]'))
+ assert_that(first_index_atag, not_none())
+ assert_that(first_index_atag['name'], is_not(
+ 'item1'), 'first item index should not be 1')
+ assert_that(first_index_atag.string, equal_to('[976]'))
+
+ def test_paging_925(self):
+ rv = self.client.get('/list/hep-ph/0901?skip=925&show=25')
+ self.assertEqual(rv.status_code, 200)
+
+ rvdata = rv.data.decode('utf-8')
+ html = BeautifulSoup(rvdata, 'html.parser')
+
+ paging = html.find(id='dlpage').find_all('div')[0]
+ assert_that(paging, not_none())
+ tgs = paging.find_all(['span', 'a'])
+ assert_that(tgs, not_none())
+ assert_that(len(tgs), 7)
+
+ assert_that(tgs[0].name, equal_to('a'))
+ assert_that(tgs[0].get_text(), equal_to('1-25'))
+
+ assert_that(tgs[1].name, equal_to('span'))
+ assert_that(tgs[1].get_text(), equal_to('...'))
+
+ assert_that(tgs[2].name, equal_to('a'))
+ assert_that(tgs[2].get_text(), equal_to('851-875'))
+
+ assert_that(tgs[3].name, equal_to('a'))
+ assert_that(tgs[3].get_text(), equal_to('876-900'))
+
+ assert_that(tgs[4].name, equal_to('a'))
+ assert_that(tgs[4].get_text(), equal_to('901-925'))
+
+ assert_that(tgs[5].name, equal_to('span'))
+ assert_that(tgs[5].get_text(), equal_to('926-950'))
+
+ assert_that(tgs[6].name, equal_to('a'))
+ assert_that(tgs[6].get_text(), equal_to('951-975'))
+
+ assert_that(tgs[7].name, equal_to('a'))
+ assert_that(tgs[7].get_text(), equal_to('976-1000'))
+
+ assert_that(tgs[8].name, equal_to('a'))
+ assert_that(tgs[8].get_text(), equal_to('1001-1001'))
+
+ # find the first article index tag
+ first_index_atag = html.find(id='articles').find_all(
+ 'dt')[0].find('a', string=re.compile(r'\[\d*\]'))
+ assert_that(first_index_atag, not_none())
+ assert_that(first_index_atag['name'], is_not(
+ 'item1'), 'first item index should not be 1')
+ assert_that(first_index_atag.string, equal_to('[926]'))
+
+ def test_odd_requests(self):
+ rv = self.client.get('/list/hep-ph/0901?skip=925&show=1000000')
+ self.assertEqual(rv.status_code, 200)
+
+ rv = self.client.get('/list/hep-ph/bogusTimePeriod')
+ self.assertNotEqual(rv.status_code, 200)
+
+ rv = self.client.get('/list/junkarchive')
+ self.assertNotEqual(rv.status_code, 200)
+
+ rv = self.client.get('/list/ao-si/0901?skip=925&show=25')
+ self.assertNotEqual(rv.status_code, 200)
+
+ rv = self.client.get('/list/math/0100')
+ self.assertNotEqual(rv.status_code, 200)
+
+ rv = self.client.get('/list/math/0113')
+ self.assertNotEqual(rv.status_code, 200)
+
+ rv = self.client.get('/list/math/0199')
+ self.assertNotEqual(rv.status_code, 200)
+
+ rv = self.client.get('/list/math/200199')
+ self.assertNotEqual(rv.status_code, 200)
+
+ rv = self.client.get('/list/math/2')
+ self.assertNotEqual(rv.status_code, 200)
+
+ rv = self.client.get('/list/math/2001999999')
+ self.assertNotEqual(rv.status_code, 200)
+
+ def test_abs_service(self):
+ service = ListingService()
+ assert_that(calling(service.list_articles_by_year).with_args('a', 1, 1, 1, 1),
+ raises(NotImplementedError))
+ assert_that(calling(service.list_articles_by_month).with_args('a', 1, 1, 1, 1),
+ raises(NotImplementedError))
+ assert_that(calling(service.list_new_articles).with_args('a', 1, 1),
+ raises(NotImplementedError))
+ assert_that(calling(service.list_pastweek_articles).with_args('a', 1, 1),
+ raises(NotImplementedError))
+
+ assert_that(service.version(), is_not(None))
+
+ def test_not_modified_from_listing_service(self):
+ with self.app.app_context():
+ flservice = get_listing_service()
+ flservice.list_new_articles = MagicMock(return_value={'not_modified': True,
+ 'expires': 'Wed, 21 Oct 2015 07:28:00 GMT'})
+ rv = self.client.get('/list/hep-ph/new')
+ self.assertEqual(
+ rv.status_code, 304, '/list controller should return 304 when service indicates not-modified')
+
+ flservice.list_pastweek_articles = MagicMock(return_value={'not_modified': True,
+ 'expires': 'Wed, 21 Oct 2015 07:28:00 GMT'})
+ rv = self.client.get('/list/hep-ph/recent')
+ self.assertEqual(
+ rv.status_code, 304, '/list controller should return 304 when service indicates not-modified')
+ rv = self.client.get('/list/hep-ph/pastweek')
+ self.assertEqual(
+ rv.status_code, 304, '/list controller should return 304 when service indicates not-modified')
+
+ flservice.list_articles_by_month = MagicMock(return_value={'not_modified': True,
+ 'expires': 'Wed, 21 Oct 2015 07:28:00 GMT'})
+ rv = self.client.get('/list/hep-ph/1801')
+ self.assertEqual(
+ rv.status_code, 304, '/list controller should return 304 when service indicates not-modified')
+
+ flservice.list_articles_by_year = MagicMock(return_value={'not_modified': True,
+ 'expires': 'Wed, 21 Oct 2015 07:28:00 GMT'})
+ rv = self.client.get('/list/hep-ph/18')
+ self.assertEqual(
+ rv.status_code, 304, '/list controller should return 304 when service indicates not-modified')
+
+ def test_list_called_from_archive(self):
+ rv = self.client.get('/list/?archive=hep-ph&year=08&month=03&submit=Go')
+ self.assertEqual(rv.status_code, 200)
+
+ rv = self.client.get('/list/?archive=hep-ph&year=08&month=all&submit=Go')
+ self.assertEqual(rv.status_code, 200)
diff --git a/tests/test_search_authors.py b/tests/test_search_authors.py
index 8fed019dc..b46be69c1 100644
--- a/tests/test_search_authors.py
+++ b/tests/test_search_authors.py
@@ -3,14 +3,14 @@
from unittest import TestCase
from browse.domain import metadata
-from browse.services.document.author_affil import split_authors
+from arxiv.util.authors import split_authors
from browse.services.document.metadata import AbsMetaSession
from browse.services.search.search_authors import queries_for_authors, split_long_author_list
from tests import path_of_for_test
class TestAuthorLinkCreation(TestCase):
-
+
def test_basic(self):
out = queries_for_authors('')
self.assertIsInstance(out, list)
diff --git a/tests/test_tex2utf.py b/tests/test_tex2utf.py
deleted file mode 100644
index 5ec27cd24..000000000
--- a/tests/test_tex2utf.py
+++ /dev/null
@@ -1,204 +0,0 @@
-"""Tests for Tex to UTF8 parsing."""
-from unittest import TestCase
-
-from browse.services.util.tex2utf import tex2utf
-
-
-class TextTex2Utf(TestCase):
-
- def test_tex2utf(self):
- test_str = "abc def ghijk lmnop qrs tuv wxyz 1234567890 !@# $%^ &* () _-=+"
- utf_out = tex2utf(test_str)
- self.assertEqual(utf_out, test_str)
-
- utf_out = tex2utf(test_str, letters=False)
- self.assertEqual(utf_out, test_str)
-
- test_str = "\\'e"
- utf_out = tex2utf(test_str)
- self.assertEqual(utf_out, chr(0xe9))
- #\'e tex2UTF é \'E tex2UTF É \'{e} tex2UTF é {\'e} tex2UTF {é}
Attention Readers: arXiv will be unavailable due to maintenance ~20 minutes starting 05:00 ET (09:00 UTC) on Thursday, November 1, 2018.
-
+
diff --git a/browse/templates/year.html b/browse/templates/year.html
new file mode 100644
index 000000000..d3c205c10
--- /dev/null
+++ b/browse/templates/year.html
@@ -0,0 +1,54 @@
+{% extends "base.html" %}
+
+{% block title %}{{archive.name}}{% endblock %}
+
+
+{% block header_h1 %}
+
+{{ config['BROWSE_SITE_LABEL'] }} > {{archive_id}} > {{year}} +
+{% endblock %} + + +{% block content %} +{{archive.name}}
+Article statistics for {{year}}
+Available montly lists with counts of {{archive_id}} articles + +cross-listings to {{archive_id}} in {{year}} +(each '|' represents 20 articles):
+ +TODO fake data as of 2019-03-04
+ +-
+ {% for month in months %}
+
- + {{month['yymm']}} + + {% for txt, url in month['art'] -%} + {%- if url -%} + {{txt}} + {%- else -%} + {{txt}} + {%- endif -%} + {%- endfor %} + {{month['new']|default('0',true)}} + {{month['cross']|default('0',true)}} + + {% endfor %} +
{{year}} totals: {{listing['new_count']|default('unknown')}} articles + {{listing['cross_count']|default('unknown')}} cross-lists
+ +Other years: +
+ {% for url, year in stats_by_year %} + {% if url %} + {{year}} + {% else %} + {{year}} + {% endif %} + {% endfor %} +
element')
+ self.assertEquals(h2_txt, 'Trackbacks for 0808.4142')
+ tb_a_tags = html.find_all('a', 'mathjax', rel='external nofollow')
+ self.assertGreater(len(tb_a_tags), 1,
+ 'There should be more than one tag for trackbacks')
+ h1_elmt = html.find('div', id='abs')
+ h1_txt = h1_elmt.get_text()
+ self.assertTrue(h1_elmt, 'Should have element')
+ self.assertRegex(
+ h1_txt,
+ r'Observation of the doubly strange b baryon Omega_b-',
+ ' element contains title of article')
+
+ def test_tb_recent(self):
+ """Test the /tb/recent page."""
+ rv = self.app.get('/tb/recent')
+ self.assertEqual(rv.status_code, 200)
+
+ rv = self.app.post('/tb/recent', data=dict(views='50'))
+ self.assertEqual(rv.status_code, 200, 'POST with integer OK')
+
+ rv = self.app.post('/tb/recent', data=dict(views='bar'))
+ self.assertEqual(rv.status_code, 400, 'POST with non-integer not OK')
+
+ rv = self.app.get('/tb/recent/foo')
+ self.assertEqual(rv.status_code, 404)
+
+ rv = self.app.post('/tb/recent', data=dict(views='1'))
+ self.assertEqual(rv.status_code, 200, 'POST with views==1 OK')
+ html = BeautifulSoup(rv.data.decode('utf-8'), 'html.parser')
+ tb_a_tags = html.find_all('a', 'mathjax', rel='external nofollow')
+ self.assertEquals(len(tb_a_tags), 1,
+ 'There should be exactly one trackback link')
+
+ def test_stats_today(self):
+ """Test the /stats/today page."""
+ rv = self.app.get('/stats/today')
+ self.assertEqual(rv.status_code, 200)
+ rv = self.app.get('/stats/today?date=20190102')
+ self.assertEqual(rv.status_code, 200)
+ html = BeautifulSoup(rv.data.decode('utf-8'), 'html.parser')
+
+ csv_dl_elmt = html.find('a', {'href': '/stats/get_hourly?date=20190102'})
+ self.assertIsNotNone(csv_dl_elmt,
+ 'csv download link exists')
+
+ def test_stats_monthly_downloads(self):
+ """Test the /stats/monthly_downloads page."""
+ rv = self.app.get('/stats/monthly_downloads')
+ self.assertEqual(rv.status_code, 200)
+ html = BeautifulSoup(rv.data.decode('utf-8'), 'html.parser')
+
+ csv_dl_elmt = html.find('a', {'href': '/stats/get_monthly_downloads'})
+ self.assertIsNotNone(csv_dl_elmt,
+ 'csv download link exists')
+
+ def test_stats_monthly_submissions(self):
+ """Test the /stats/monthly_submissions page."""
+ rv = self.app.get('/stats/monthly_submissions')
+ self.assertEqual(rv.status_code, 200)
+ html = BeautifulSoup(rv.data.decode('utf-8'), 'html.parser')
+
+ csv_dl_elmt = html.find('a', {'href': '/stats/get_monthly_submissions'})
+ self.assertIsNotNone(csv_dl_elmt,
+ 'csv download link exists')
+
+
def test_abs_without_license_field(self):
f1 = ABS_FILES + '/ftp/arxiv/papers/0704/0704.0001.abs'
m = AbsMetaSession.parse_abs_file(filename=f1)
@@ -148,7 +247,8 @@ def test_requested_version(self):
self.assertIsNotNone(pdf_dl_elmt,
'pdf download link without version affix exists')
pdf_dl_elmt = html.find('a', {'href': '/pdf/physics/9707012v'})
- self.assertIsNone(pdf_dl_elmt, 'pdf download link with version affix does not exist')
+ self.assertIsNone(
+ pdf_dl_elmt, 'pdf download link with version affix does not exist')
rv = self.app.get('/abs/physics/9707012v4')
self.assertEqual(rv.status_code, 200)
@@ -200,7 +300,6 @@ def test_1501_9999(self):
'href="ftp://ftp.arxiv.org/cheese.txt"' in rv.data.decode('utf-8'),
"FTP URLs should be turned into links ARXIVNG-1242")
-
def test_160408245(self):
"""Test linking in 1604.08245."""
id = '1604.08245'
@@ -234,62 +333,65 @@ def test_arxivng_1246(self):
def test_authors_and_arxivId_in_title(self):
id = '1501.99999'
- rv = self.app.get('/abs/'+id)
+ rv = self.app.get('/abs/' + id)
self.assertEqual(rv.status_code, 200)
html = BeautifulSoup(rv.data.decode('utf-8'), 'html.parser')
- title_elmt = html.find('h1','title')
- self.assertTrue(title_elmt,'Should title element')
+ title_elmt = html.find('h1', 'title')
+ self.assertTrue(title_elmt, 'Should title element')
ida = title_elmt.find('a')
self.assertTrue(ida, 'Should be tag in title')
- self.assertIsNotNone(ida['href'],' tag in title should have href')
- self.assertEqual(ida['href'], '/abs/1501.99998')
+ self.assertIsNotNone(ida['href'], ' tag in title should have href')
+ self.assertEqual(ida['href'], 'https://arxiv.org/abs/1501.99998')
+
self.assertEqual(ida.text, '1501.99998')
- au_a_tags = html.find('div','authors').find_all('a')
- self.assertGreater(len(au_a_tags), 1, 'Should be some a tags for authors')
+ au_a_tags = html.find('div', 'authors').find_all('a')
+ self.assertGreater(len(au_a_tags), 1,
+ 'Should be some a tags for authors')
self.assertNotIn('query=The', au_a_tags[0]['href'],
'Collaboration author query should not have "The"')
self.assertEqual(au_a_tags[0].text, 'SuperSuper Collaboration')
-
def test_long_author_colab(self):
id = '1501.05201'
- rv = self.app.get('/abs/'+id)
+ rv = self.app.get('/abs/' + id)
self.assertEqual(rv.status_code, 200)
html = BeautifulSoup(rv.data.decode('utf-8'), 'html.parser')
- auths_elmt = html.find('div','authors')
- self.assertTrue(auths_elmt,'Should authors div element')
+ auths_elmt = html.find('div', 'authors')
+ self.assertTrue(auths_elmt, 'Should authors div element')
a_tags = auths_elmt.find_all('a')
- self.assertEqual(len(a_tags), 2, 'Should be two tags in authors div')
+ self.assertEqual(
+ len(a_tags), 2, 'Should be two tags in authors div')
- colab=a_tags[1]
+ colab = a_tags[1]
- self.assertIsNotNone(colab['href'],' tag in title should have href')
- self.assertEqual(colab['href'], 'https://arxiv.org/search/physics?searchtype=author&query=ILL%2FESS%2FLiU+collaboration')
- self.assertEqual(colab.text, 'ILL/ESS/LiU collaboration for the development of the B10 detector technology in the framework of the CRISP project')
+ self.assertIsNotNone(
+ colab['href'], ' tag in title should have href')
+ self.assertEqual(
+ colab['href'], 'https://arxiv.org/search/physics?searchtype=author&query=ILL%2FESS%2FLiU+collaboration')
+ self.assertEqual(
+ colab.text, 'ILL/ESS/LiU collaboration for the development of the B10 detector technology in the framework of the CRISP project')
-
- @unittest.skip("In current implementation, conflicts with comma test below.")
+ @unittest.skip("In current implementation, conflicts with comma test below.")
def test_space_in_author_list(self):
id = '1210.8438'
- rv = self.app.get('/abs/'+id)
+ rv = self.app.get('/abs/' + id)
self.assertEqual(rv.status_code, 200)
html = BeautifulSoup(rv.data.decode('utf-8'), 'html.parser')
- auths_elmt = html.find('div','authors')
- self.assertTrue(auths_elmt,'Should authors div element')
+ auths_elmt = html.find('div', 'authors')
+ self.assertTrue(auths_elmt, 'Should authors div element')
self.assertIn('Zhe (Rita) Liang,', auths_elmt.text,
'Should be a space after (Rita)')
-
def test_comma_in_author_list(self):
id = '0704.0155'
- rv = self.app.get('/abs/'+id)
+ rv = self.app.get('/abs/' + id)
self.assertEqual(rv.status_code, 200)
html = BeautifulSoup(rv.data.decode('utf-8'), 'html.parser')
auths_elmt = html.find('div', 'authors')
@@ -313,5 +415,34 @@ def test_psi_in_abs(self):
self.assertIn('The phase difference $\phi$, between the superconducting',
abs_elmt.text,
"Expecting uncoverted $\phi$ in html abstract.")
-
+ def test_year(self):
+ rv = self.app.get('/year/astro-ph/09')
+ self.assertEqual(rv.status_code, 200)
+
+ rv = self.app.get('/year/astro-ph/')
+ self.assertEqual( rv.status_code, 200)
+
+ rv = self.app.get('/year/astro-ph')
+ self.assertEqual( rv.status_code, 200)
+
+ rv = self.app.get('/year/astro-ph/09/')
+ self.assertEqual(rv.status_code, 200)
+
+ rv = self.app.get('/year')
+ self.assertEqual( rv.status_code, 404)
+
+ rv = self.app.get('/year/astro-ph/9999')
+ self.assertEqual(rv.status_code, 307, 'Future year should cause temporary redirect')
+
+ rv = self.app.get('/year/fakearchive/01')
+ self.assertNotEqual(rv.status_code, 200)
+ self.assertLess( rv.status_code, 500, 'should not cause a 5XX')
+
+ rv = self.app.get('/year/002/0000')
+ self.assertLess( rv.status_code, 500, 'should not cause a 5XX')
+
+ rv = self.app.get('/year/astro-py/9223372036854775808')
+ self.assertLess( rv.status_code, 500, 'should not cause a 5XX')
+
+
diff --git a/tests/test_categories.py b/tests/test_categories.py
index 00a588420..a05f0b99f 100644
--- a/tests/test_categories.py
+++ b/tests/test_categories.py
@@ -3,7 +3,7 @@
from tests.test_abs_parser import ABS_FILES
from browse.services.document.metadata import AbsMetaSession
-
+from browse.domain.category import Category
class CategoriesTest(unittest.TestCase):
@@ -15,7 +15,7 @@ def test_categories_for_0906_3421v1_cats(self):
['Statistical Mechanics (cond-mat.stat-mech)', 'Mathematical Physics (math-ph)'])
doc = self.absService.get_abs(id)
self.assertIsNotNone(doc)
- assert_that(doc.primary_category.display_str(), equal_to(primary))
+ assert_that(doc.primary_category.display, equal_to(primary))
assert_that(doc.display_secondaries(), equal_to(secondaries))
def test_categories_for_0906_3421_cats(self):
@@ -23,7 +23,7 @@ def test_categories_for_0906_3421_cats(self):
['Statistical Mechanics (cond-mat.stat-mech)', 'Mathematical Physics (math-ph)'])
doc = self.absService.get_abs(id)
self.assertIsNotNone(doc)
- assert_that(doc.primary_category.display_str(), equal_to(primary))
+ assert_that(doc.primary_category.display, equal_to(primary))
assert_that(doc.display_secondaries(), equal_to(secondaries))
def test_categories_for_0704_0129_cats(self):
@@ -31,7 +31,7 @@ def test_categories_for_0704_0129_cats(self):
['Mathematical Physics (math-ph)', 'Analysis of PDEs (math.AP)'])
doc = self.absService.get_abs(id)
self.assertIsNotNone(doc)
- assert_that(doc.primary_category.display_str(), equal_to(primary))
+ assert_that(doc.primary_category.display, equal_to(primary))
assert_that(doc.display_secondaries(), equal_to(secondaries))
def test_categories_for_0704_0914_cats(self):
@@ -39,7 +39,7 @@ def test_categories_for_0704_0914_cats(self):
['Mathematical Physics (math-ph)', 'Optics (physics.optics)'])
doc = self.absService.get_abs(id)
self.assertIsNotNone(doc)
- assert_that(doc.primary_category.display_str(), equal_to(primary))
+ assert_that(doc.primary_category.display, equal_to(primary))
assert_that(doc.display_secondaries(), equal_to(secondaries))
def test_categories_for_0704_0582_cats(self):
@@ -47,7 +47,7 @@ def test_categories_for_0704_0582_cats(self):
'Probability (math.PR)', ['Mathematical Physics (math-ph)'])
doc = self.absService.get_abs(id)
self.assertIsNotNone(doc)
- assert_that(doc.primary_category.display_str(), equal_to(primary))
+ assert_that(doc.primary_category.display, equal_to(primary))
assert_that(doc.display_secondaries(), equal_to(secondaries))
def test_categories_for_0704_0495_cats(self):
@@ -55,7 +55,7 @@ def test_categories_for_0704_0495_cats(self):
['Mathematical Physics (math-ph)'])
doc = self.absService.get_abs(id)
self.assertIsNotNone(doc)
- assert_that(doc.primary_category.display_str(), equal_to(primary))
+ assert_that(doc.primary_category.display, equal_to(primary))
assert_that(doc.display_secondaries(), equal_to(secondaries))
def test_categories_for_0704_0681_cats(self):
@@ -63,7 +63,7 @@ def test_categories_for_0704_0681_cats(self):
['Statistical Mechanics (cond-mat.stat-mech)', 'Instrumentation and Detectors (physics.ins-det)', 'Optics (physics.optics)'])
doc = self.absService.get_abs(id)
self.assertIsNotNone(doc)
- assert_that(doc.primary_category.display_str(), equal_to(primary))
+ assert_that(doc.primary_category.display, equal_to(primary))
assert_that(doc.display_secondaries(), equal_to(secondaries))
def test_categories_for_0704_0761_cats(self):
@@ -71,7 +71,7 @@ def test_categories_for_0704_0761_cats(self):
['Soft Condensed Matter (cond-mat.soft)', 'Chemical Physics (physics.chem-ph)'])
doc = self.absService.get_abs(id)
self.assertIsNotNone(doc)
- assert_that(doc.primary_category.display_str(), equal_to(primary))
+ assert_that(doc.primary_category.display, equal_to(primary))
assert_that(doc.display_secondaries(), equal_to(secondaries))
def test_categories_for_0704_0528_cats(self):
@@ -79,7 +79,7 @@ def test_categories_for_0704_0528_cats(self):
['Information Theory (cs.IT)'])
doc = self.absService.get_abs(id)
self.assertIsNotNone(doc)
- assert_that(doc.primary_category.display_str(), equal_to(primary))
+ assert_that(doc.primary_category.display, equal_to(primary))
assert_that(doc.display_secondaries(), equal_to(secondaries))
def test_categories_for_0704_0869_cats(self):
@@ -87,7 +87,7 @@ def test_categories_for_0704_0869_cats(self):
['Mathematical Physics (math-ph)'])
doc = self.absService.get_abs(id)
self.assertIsNotNone(doc)
- assert_that(doc.primary_category.display_str(), equal_to(primary))
+ assert_that(doc.primary_category.display, equal_to(primary))
assert_that(doc.display_secondaries(), equal_to(secondaries))
def test_categories_for_0704_0796_cats(self):
@@ -95,7 +95,7 @@ def test_categories_for_0704_0796_cats(self):
['Astrophysics (astro-ph)', 'Statistical Mechanics (cond-mat.stat-mech)', 'High Energy Physics - Phenomenology (hep-ph)', 'Mathematical Physics (math-ph)'])
doc = self.absService.get_abs(id)
self.assertIsNotNone(doc)
- assert_that(doc.primary_category.display_str(), equal_to(primary))
+ assert_that(doc.primary_category.display, equal_to(primary))
assert_that(doc.display_secondaries(), equal_to(secondaries))
def test_categories_for_0704_0046_cats(self):
@@ -103,7 +103,7 @@ def test_categories_for_0704_0046_cats(self):
'Quantum Physics (quant-ph)', ['Information Theory (cs.IT)'])
doc = self.absService.get_abs(id)
self.assertIsNotNone(doc)
- assert_that(doc.primary_category.display_str(), equal_to(primary))
+ assert_that(doc.primary_category.display, equal_to(primary))
assert_that(doc.display_secondaries(), equal_to(secondaries))
def test_categories_for_0704_0976_cats(self):
@@ -112,7 +112,7 @@ def test_categories_for_0704_0976_cats(self):
'Chaotic Dynamics (nlin.CD)'])
doc = self.absService.get_abs(id)
self.assertIsNotNone(doc)
- assert_that(doc.primary_category.display_str(), equal_to(primary))
+ assert_that(doc.primary_category.display, equal_to(primary))
assert_that(doc.display_secondaries(), equal_to(secondaries))
def test_categories_for_0704_0041_cats(self):
@@ -120,7 +120,7 @@ def test_categories_for_0704_0041_cats(self):
['Mathematical Physics (math-ph)'])
doc = self.absService.get_abs(id)
self.assertIsNotNone(doc)
- assert_that(doc.primary_category.display_str(), equal_to(primary))
+ assert_that(doc.primary_category.display, equal_to(primary))
assert_that(doc.display_secondaries(), equal_to(secondaries))
def test_categories_for_0704_0918_cats(self):
@@ -128,7 +128,7 @@ def test_categories_for_0704_0918_cats(self):
'Algebraic Geometry (math.AG)', ['Statistics Theory (math.ST)'])
doc = self.absService.get_abs(id)
self.assertIsNotNone(doc)
- assert_that(doc.primary_category.display_str(), equal_to(primary))
+ assert_that(doc.primary_category.display, equal_to(primary))
assert_that(doc.display_secondaries(), equal_to(secondaries))
def test_categories_for_0704_0123_cats(self):
@@ -136,7 +136,7 @@ def test_categories_for_0704_0123_cats(self):
['Other Condensed Matter (cond-mat.other)', 'Optics (physics.optics)'])
doc = self.absService.get_abs(id)
self.assertIsNotNone(doc)
- assert_that(doc.primary_category.display_str(), equal_to(primary))
+ assert_that(doc.primary_category.display, equal_to(primary))
assert_that(doc.display_secondaries(), equal_to(secondaries))
def test_categories_for_0704_0520_cats(self):
@@ -145,7 +145,7 @@ def test_categories_for_0704_0520_cats(self):
'Atomic Physics (physics.atom-ph)'])
doc = self.absService.get_abs(id)
self.assertIsNotNone(doc)
- assert_that(doc.primary_category.display_str(), equal_to(primary))
+ assert_that(doc.primary_category.display, equal_to(primary))
assert_that(doc.display_secondaries(), equal_to(secondaries))
def test_categories_for_0704_0084_cats(self):
@@ -154,7 +154,7 @@ def test_categories_for_0704_0084_cats(self):
'Fluid Dynamics (physics.flu-dyn)'])
doc = self.absService.get_abs(id)
self.assertIsNotNone(doc)
- assert_that(doc.primary_category.display_str(), equal_to(primary))
+ assert_that(doc.primary_category.display, equal_to(primary))
assert_that(doc.display_secondaries(), equal_to(secondaries))
def test_categories_for_0704_0588_cats(self):
@@ -162,7 +162,7 @@ def test_categories_for_0704_0588_cats(self):
'Probability (math.PR)', ['Statistics Theory (math.ST)'])
doc = self.absService.get_abs(id)
self.assertIsNotNone(doc)
- assert_that(doc.primary_category.display_str(), equal_to(primary))
+ assert_that(doc.primary_category.display, equal_to(primary))
assert_that(doc.display_secondaries(), equal_to(secondaries))
def test_categories_for_0704_0687_cats(self):
@@ -170,5 +170,17 @@ def test_categories_for_0704_0687_cats(self):
['Mathematical Physics (math-ph)'])
doc = self.absService.get_abs(id)
self.assertIsNotNone(doc)
- assert_that(doc.primary_category.display_str(), equal_to(primary))
+ assert_that(doc.primary_category.display, equal_to(primary))
assert_that(doc.display_secondaries(), equal_to(secondaries))
+
+
+ def test_non_tax_cats(self):
+ cat = Category('nontx.AL')
+ assert_that(cat.display , equal_to('nontx.AL'))
+
+ # TODO: confirm actual desired behavior
+ # cat = Category('math.XX')
+ # assert_that(cat.display , equal_to('Mathematics (math.XX)'))
+
+ cat = Category('notaxArc')
+ assert_that(cat.display , equal_to('notaxArc'))
diff --git a/tests/test_click_through.py b/tests/test_click_through.py
deleted file mode 100644
index a6a6fd3f5..000000000
--- a/tests/test_click_through.py
+++ /dev/null
@@ -1,15 +0,0 @@
-# from unittest import TestCase
-import unittest
-
-from hypothesis import given
-from hypothesis.strategies import text
-
-from browse.util.clickthrough import create_hash, is_hash_valid
-
-
-class TestClickthrough(unittest.TestCase):
-
- @given(text(), text())
- def test_clickthrough(self, s, v):
- h = create_hash(s, v)
- self.assertTrue(is_hash_valid(s, v, h))
diff --git a/tests/test_cookies.py b/tests/test_cookies.py
new file mode 100644
index 000000000..db1b8b22b
--- /dev/null
+++ b/tests/test_cookies.py
@@ -0,0 +1,37 @@
+"""test cookies"""
+import unittest
+
+#from bs4 import BeautifulSoup
+
+from app import app
+
+
+class CookiesPageTest(unittest.TestCase):
+
+ def setUp(self):
+ app.testing = True
+ app.config['APPLICATION_ROOT'] = ''
+ self.app = app.test_client()
+
+
+ def test_cookies_with_no_params(self):
+ """Test the cookies page."""
+ rv = self.app.get('/cookies')
+ self.assertEqual(rv.status_code, 200)
+ html = rv.data.decode('utf-8')
+ self.assertIn('Select preferred download format', html)
+ self.assertIn('show additional debugging information', html, 'should have SHOW debugging link')
+
+ def test_cookies_with_debug(self):
+ """Test the cookies page."""
+ rv = self.app.get('/cookies?debug=1')
+ self.assertEqual(rv.status_code, 200)
+ html = rv.data.decode('utf-8')
+ self.assertIn('Select preferred download format', html)
+ self.assertIn('hide debugging information', html, 'should have HIDE debugging link')
+
+ def test_post_to_cookies(self):
+ rv = self.app.post('/cookies/set?debug=1', data={'ps':'pdf'})
+ self.assertEqual(rv.status_code, 302)
+ cookies = map(lambda kv: kv[1], filter(lambda kv : kv[0]=='Set-Cookie', rv.headers.items()))
+ self.assertIn('xxx-ps-defaults=pdf; Path=/', cookies)
diff --git a/tests/test_database_service.py b/tests/test_database_service.py
index 025608bda..f6d72e62f 100644
--- a/tests/test_database_service.py
+++ b/tests/test_database_service.py
@@ -4,9 +4,11 @@
from unittest import mock, TestCase
from unittest.mock import Mock, patch
from sqlalchemy.exc import SQLAlchemyError
-
from sqlalchemy.orm.exc import NoResultFound
+from browse.services.database.models import TrackbackPing
+from browse.domain.identifier import Identifier
+
from tests import grep_f_count, execute_sql_files, path_of_for_test
@@ -82,6 +84,10 @@ def setUpClass(cls) -> None:
execute_sql_files(sql_files, database.db.engine)
database.db.session.commit()
+ """Disable logging to avoid messy output during testing"""
+ import logging
+ logging.disable(logging.WARNING)
+
def test_get_institution_returns_a_label(self) -> None:
"""If IP address matches an institution, a label is returned."""
label = TestBrowseDatabaseService.database_service.get_institution(
@@ -123,7 +129,8 @@ def test_get_institution_returns_a_label(self) -> None:
def test_all_trackback_pings(self) -> None:
"""Test if all trackback pings are counted."""
- doc_sql_file = path_of_for_test('data/db/sql/arXiv_trackback_pings.sql')
+ doc_sql_file = path_of_for_test(
+ 'data/db/sql/arXiv_trackback_pings.sql')
count_from_file = grep_f_count(
doc_sql_file,
@@ -151,14 +158,14 @@ def test_all_trackback_pings(self) -> None:
)
def test_trackback_pings(self) -> None:
- """Test if trackback pings for specific paper are counted."""
+ """Test if trackback pings for a specific paper are counted."""
test_paper_id = '0808.4142'
count_from_db: int = TestBrowseDatabaseService.database_service\
.count_trackback_pings(test_paper_id)
count_from_db_list: int = TestBrowseDatabaseService.database_service\
- .get_trackback_pings(test_paper_id).__len__()
+ .get_paper_trackback_pings(test_paper_id).__len__()
self.assertEqual(
- count_from_db, 8,
+ count_from_db, 9,
f'Correct count of pings returned for paper {test_paper_id}'
)
self.assertEqual(
@@ -166,6 +173,21 @@ def test_trackback_pings(self) -> None:
f'Correct count of pings returned for paper {test_paper_id}'
)
+ def test_recent_trackback_pings(self) -> None:
+ """Test if recent trackbacks can be retrieved."""
+ tbs: List = TestBrowseDatabaseService.database_service.\
+ get_recent_trackback_pings(max_trackbacks=-1)
+ self.assertEqual(len(tbs), 0, 'List should be empty')
+ tbs: List = TestBrowseDatabaseService.database_service.\
+ get_recent_trackback_pings(max_trackbacks=25)
+ self.assertGreater(len(tbs), 0, 'List should be nonempty')
+ for tb in tbs:
+ self.assertIsInstance(tb[0], TrackbackPing)
+ self.assertIsInstance(tb[1], str)
+ self.assertIsInstance(Identifier(
+ arxiv_id=tb[1]), Identifier, 'Value looks like an Identifier')
+ self.assertIsInstance(tb[2], str)
+
def test_sciencewise_ping(self) -> None:
"""Test whether paper with version suffix has a ScienceWISE ping."""
test_paper_id_v = '1605.09669v2'
@@ -209,15 +231,23 @@ def test_get_dblp_authors(self) -> None:
TestBrowseDatabaseService.database_service.get_dblp_authors(
test_paper_id), [])
+ def test_get_document_count(self) -> None:
+ """Test document count function."""
+ self.assertGreater(
+ TestBrowseDatabaseService.database_service.get_document_count(),
+ 0,
+ 'There is at least one document in the DB.'
+ )
+
@mock.patch('browse.services.database.models.db.session.query')
- def test_error_conditions(self, mock_query)->None:
+ def test_error_conditions(self, mock_query) -> None:
mock_query.side_effect = NoResultFound
self.assertEqual(
TestBrowseDatabaseService.database_service.get_institution('10.0.0.1'), None)
self.assertEqual([],
- TestBrowseDatabaseService.database_service.get_all_trackback_pings())
+ TestBrowseDatabaseService.database_service.get_all_trackback_pings())
self.assertListEqual(
- TestBrowseDatabaseService.database_service.get_trackback_pings('0704.0361'), [])
+ TestBrowseDatabaseService.database_service.get_paper_trackback_pings('0704.0361'), [])
self.assertEqual(
TestBrowseDatabaseService.database_service.count_trackback_pings('0704.0361'), 0)
self.assertEqual(
@@ -231,12 +261,18 @@ def test_error_conditions(self, mock_query)->None:
mock_query.side_effect = SQLAlchemyError
self.assertRaises(SQLAlchemyError,
TestBrowseDatabaseService.database_service.get_institution, '10.0.0.1')
- self.assertRaises(SQLAlchemyError, TestBrowseDatabaseService.database_service.get_all_trackback_pings)
- self.assertRaises(SQLAlchemyError, TestBrowseDatabaseService.database_service.get_trackback_pings, 'paperx')
- self.assertRaises(SQLAlchemyError, TestBrowseDatabaseService.database_service.count_all_trackback_pings)
- self.assertRaises(SQLAlchemyError, TestBrowseDatabaseService.database_service.has_sciencewise_ping, 'px')
- self.assertRaises(SQLAlchemyError, TestBrowseDatabaseService.database_service.get_dblp_listing_path, 'px')
- self.assertRaises(SQLAlchemyError, TestBrowseDatabaseService.database_service.get_dblp_authors,'authx')
+ self.assertRaises(
+ SQLAlchemyError, TestBrowseDatabaseService.database_service.get_all_trackback_pings)
+ self.assertRaises(
+ SQLAlchemyError, TestBrowseDatabaseService.database_service.get_paper_trackback_pings, 'paperx')
+ self.assertRaises(
+ SQLAlchemyError, TestBrowseDatabaseService.database_service.count_all_trackback_pings)
+ self.assertRaises(
+ SQLAlchemyError, TestBrowseDatabaseService.database_service.has_sciencewise_ping, 'px')
+ self.assertRaises(
+ SQLAlchemyError, TestBrowseDatabaseService.database_service.get_dblp_listing_path, 'px')
+ self.assertRaises(
+ SQLAlchemyError, TestBrowseDatabaseService.database_service.get_dblp_authors, 'authx')
@classmethod
def tearDownClass(cls) -> None:
diff --git a/tests/test_doi.py b/tests/test_doi.py
index 124106d76..18772845b 100644
--- a/tests/test_doi.py
+++ b/tests/test_doi.py
@@ -13,6 +13,7 @@
from app import app
+@unittest.skip('We will move this test and any required test data to arxiv-base in the near future')
class DoiTest(unittest.TestCase):
def setUp(self):
diff --git a/tests/test_exceptions.py b/tests/test_exceptions.py
index 54186696b..2ddc2e6d0 100644
--- a/tests/test_exceptions.py
+++ b/tests/test_exceptions.py
@@ -19,6 +19,8 @@ def setUp(self):
wlog.disabled = True
self.app = create_web_app()
+ self.app.testing = True
+ self.app.config['APPLICATION_ROOT'] = ''
self.client = self.app.test_client()
def test_404(self):
@@ -54,7 +56,8 @@ def test_500(self, mock_abs):
"""Disable logging to avoid messy output during testing"""
self.app.logger.disabled = True
- response = self.client.get('/abs/1234.5678')
- self.assertEqual(response.status_code,
- status.HTTP_500_INTERNAL_SERVER_ERROR)
- self.assertIn('text/html', response.content_type)
+ with self.assertRaises(AbsException):
+ response = self.client.get('/abs/1234.5678')
+ self.assertEqual(response.status_code,
+ status.HTTP_500_INTERNAL_SERVER_ERROR)
+ self.assertIn('text/html', response.content_type)
diff --git a/tests/test_filters.py b/tests/test_filters.py
index ee38e22e1..31c7d8e00 100644
--- a/tests/test_filters.py
+++ b/tests/test_filters.py
@@ -1,5 +1,4 @@
import unittest
-from hamcrest import *
from functools import partial
from jinja2 import escape, Markup, Environment
@@ -7,229 +6,217 @@
from flask import appcontext_pushed, url_for
from app import app
-from browse.filters import line_feed_to_br, tex_to_utf, entity_to_utf
-from browse.util.id_patterns import do_dois_to_tags, do_dois_id_urls_to_tags,\
- do_id_to_tags
+from arxiv.base.urls import links, urlizer, urlize
+from arxiv.base.filters import abstract_lf_to_br, f_tex2utf
+from browse.filters import entity_to_utf
+@unittest.skip("These test features implemented in arxiv-base so move these tests to arxiv-base")
+class Jinja_Custom_Filters_Test(unittest.TestCase):
+ """Browse jinja filter tests."""
-def _id_to_url(id: str):
- return url_for('browse.abstract', arxiv_id=id)
-
-def arxiv_urlize(txt):
- return do_dois_id_urls_to_tags(_id_to_url,None,txt)
-
-def doi_urls(fn, txt):
- return do_dois_to_tags(fn, txt)
-
-def arxiv_id_urls(txt):
- return do_id_to_tags(_id_to_url, txt)
-
-class Jinja_Custom_Fitlers_Test(unittest.TestCase):
def test_with_jinja(self):
- jenv = Environment(autoescape=True)
- jenv.filters['doi_urls'] = partial(doi_urls, lambda x: x)
- assert_that(
- jenv.from_string(
- '{{"something 10.1103/PhysRevD.76.013009 or other"|doi_urls}}'
- ).render(),
- equal_to(
- 'something 10.1103/PhysRevD.76.013009 or other'
- ))
+ """Basic urlize DOI filter test."""
+ with app.app_context():
+ jenv = Environment(autoescape=True)
+ jenv.filters['urlize'] = urlizer(
+ ['doi']
+ )
+ self.assertEqual(
+ jenv.from_string(
+ '{{"something 10.1103/PhysRevD.76.013009 or other"|urlize}}'
+ ).render(),
+ 'something <a class="link-https link-external" data-doi="10.1103/PhysRevD.76.013009" href="https://arxiv.org/ct?url=https%3A%2F%2Fdx.doi.org%2F10.1103%2FPhysRevD.76.013009&v=d0670bbf" rel="external noopener nofollow">10.1103/PhysRevD.76.013009</a> or other'
+ )
def test_with_jinja_escapes(self):
- jenv = Environment(autoescape=True)
- jenv.filters['doi_urls'] = partial(doi_urls, lambda x: x)
- jenv.filters['arxiv_urlize'] = arxiv_urlize
- assert_that(
- jenv.from_string(
- '{{"something 10.1103/PhysRevD.76.013009 or other"|doi_urls}}'
- ).render(),
- equal_to(
- 'something 10.1103/PhysRevD.76.013009 or other'
- ))
-
- assert_that(
- jenv.from_string(
- '{{" something 10.1103/PhysRevD.76.013009"|arxiv_urlize}}'
- ).render(),
- equal_to(
- '<script>bad junk</script> something 10.1103/PhysRevD.76.013009'
- ))
+ """Test the tex2utf filter with jinja escapes."""
+ with app.app_context():
+ jenv = Environment(autoescape=True)
+ jenv.filters['urlize'] = urlizer(
+ ['arxiv_id', 'doi']
+ )
+
+ # TODO: urlize doesn't seem to return a Markup object?
+ self.assertEqual(
+ jenv.from_string(
+ '{{"something 10.1103/PhysRevD.76.013009 or other"|urlize}}'
+ ).render(),
+ 'something <a class="link-https link-external" data-doi="10.1103/PhysRevD.76.013009" href="https://arxiv.org/ct?url=https%3A%2F%2Fdx.doi.org%2F10.1103%2FPhysRevD.76.013009&v=d0670bbf" rel="external noopener nofollow">10.1103/PhysRevD.76.013009</a> or other'
+ )
+
+ self.assertEqual(
+ jenv.from_string(
+ '{{" something 10.1103/PhysRevD.76.013009"|urlize}}'
+ ).render(),
+ '<script>bad junk</script> something <a class="link-https link-external" data-doi="10.1103/PhysRevD.76.013009" href="https://arxiv.org/ct?url=https%3A%2F%2Fdx.doi.org%2F10.1103%2FPhysRevD.76.013009&v=d0670bbf" rel="external noopener nofollow">10.1103/PhysRevD.76.013009</a>'
+ )
def test_doi_filter(self):
- doi_fn = partial(doi_urls, lambda x: x)
-
- s = ''
- self.assertEqual(doi_fn(s), s)
-
- s = 'some test string 23$6#$5<>&456 http://google.com/notadoi'
- assert_that(doi_fn(s), equal_to(escape(s)))
-
- doi = '10.1103/PhysRevD.76.013009'
- doiurl = doi_fn(doi)
- self.assertRegex(doiurl, r'^10.1103/PhysRevD.76.013009
')))
-
- s = f'something something {doi} endthing'
- doiurl = doi_fn(s)
- self.assertRegex(doiurl, r'10.1103/PhysRevA.99.013009'
- ' 10.1103/PhysRevZ.44.023009'
- ' 10.1103/PhysRevX.90.012309'
- ' 10.1103/BioRevX.44.123456'
- )))
-
- txt = ''
- assert_that(
- doi_fn(f'{doi} {txt}'),
- equal_to(
- Markup(f'10.1103/PhysRevD.76.013009 {escape(txt)}')
- ))
+ """Test the urlizer DOI filter."""
+ with app.app_context():
+ s = 'some test string 23$6#$5<>&456 http://google.com/notadoi'
+ urlize_dois = urlizer(
+ ['doi']
+ )
+ self.assertEqual(urlize_dois(s), str(escape(s)))
+
+ doi = '10.1103/PhysRevD.76.013009'
+ doiurl = urlize_dois(doi)
+ self.assertRegex(doiurl, r'^10.1103/PhysRevD.76.013009
element contains title of article')
+
+ def test_tb_recent(self):
+ """Test the /tb/recent page."""
+ rv = self.app.get('/tb/recent')
+ self.assertEqual(rv.status_code, 200)
+
+ rv = self.app.post('/tb/recent', data=dict(views='50'))
+ self.assertEqual(rv.status_code, 200, 'POST with integer OK')
+
+ rv = self.app.post('/tb/recent', data=dict(views='bar'))
+ self.assertEqual(rv.status_code, 400, 'POST with non-integer not OK')
+
+ rv = self.app.get('/tb/recent/foo')
+ self.assertEqual(rv.status_code, 404)
+
+ rv = self.app.post('/tb/recent', data=dict(views='1'))
+ self.assertEqual(rv.status_code, 200, 'POST with views==1 OK')
+ html = BeautifulSoup(rv.data.decode('utf-8'), 'html.parser')
+ tb_a_tags = html.find_all('a', 'mathjax', rel='external nofollow')
+ self.assertEquals(len(tb_a_tags), 1,
+ 'There should be exactly one trackback link')
+
+ def test_stats_today(self):
+ """Test the /stats/today page."""
+ rv = self.app.get('/stats/today')
+ self.assertEqual(rv.status_code, 200)
+ rv = self.app.get('/stats/today?date=20190102')
+ self.assertEqual(rv.status_code, 200)
+ html = BeautifulSoup(rv.data.decode('utf-8'), 'html.parser')
+
+ csv_dl_elmt = html.find('a', {'href': '/stats/get_hourly?date=20190102'})
+ self.assertIsNotNone(csv_dl_elmt,
+ 'csv download link exists')
+
+ def test_stats_monthly_downloads(self):
+ """Test the /stats/monthly_downloads page."""
+ rv = self.app.get('/stats/monthly_downloads')
+ self.assertEqual(rv.status_code, 200)
+ html = BeautifulSoup(rv.data.decode('utf-8'), 'html.parser')
+
+ csv_dl_elmt = html.find('a', {'href': '/stats/get_monthly_downloads'})
+ self.assertIsNotNone(csv_dl_elmt,
+ 'csv download link exists')
+
+ def test_stats_monthly_submissions(self):
+ """Test the /stats/monthly_submissions page."""
+ rv = self.app.get('/stats/monthly_submissions')
+ self.assertEqual(rv.status_code, 200)
+ html = BeautifulSoup(rv.data.decode('utf-8'), 'html.parser')
+
+ csv_dl_elmt = html.find('a', {'href': '/stats/get_monthly_submissions'})
+ self.assertIsNotNone(csv_dl_elmt,
+ 'csv download link exists')
+
+
def test_abs_without_license_field(self):
f1 = ABS_FILES + '/ftp/arxiv/papers/0704/0704.0001.abs'
m = AbsMetaSession.parse_abs_file(filename=f1)
@@ -148,7 +247,8 @@ def test_requested_version(self):
self.assertIsNotNone(pdf_dl_elmt,
'pdf download link without version affix exists')
pdf_dl_elmt = html.find('a', {'href': '/pdf/physics/9707012v'})
- self.assertIsNone(pdf_dl_elmt, 'pdf download link with version affix does not exist')
+ self.assertIsNone(
+ pdf_dl_elmt, 'pdf download link with version affix does not exist')
rv = self.app.get('/abs/physics/9707012v4')
self.assertEqual(rv.status_code, 200)
@@ -200,7 +300,6 @@ def test_1501_9999(self):
'href="ftp://ftp.arxiv.org/cheese.txt"' in rv.data.decode('utf-8'),
"FTP URLs should be turned into links ARXIVNG-1242")
-
def test_160408245(self):
"""Test linking in 1604.08245."""
id = '1604.08245'
@@ -234,62 +333,65 @@ def test_arxivng_1246(self):
def test_authors_and_arxivId_in_title(self):
id = '1501.99999'
- rv = self.app.get('/abs/'+id)
+ rv = self.app.get('/abs/' + id)
self.assertEqual(rv.status_code, 200)
html = BeautifulSoup(rv.data.decode('utf-8'), 'html.parser')
- title_elmt = html.find('h1','title')
- self.assertTrue(title_elmt,'Should title element')
+ title_elmt = html.find('h1', 'title')
+ self.assertTrue(title_elmt, 'Should title element')
ida = title_elmt.find('a')
self.assertTrue(ida, 'Should be tag in title')
- self.assertIsNotNone(ida['href'],' tag in title should have href')
- self.assertEqual(ida['href'], '/abs/1501.99998')
+ self.assertIsNotNone(ida['href'], ' tag in title should have href')
+ self.assertEqual(ida['href'], 'https://arxiv.org/abs/1501.99998')
+
self.assertEqual(ida.text, '1501.99998')
- au_a_tags = html.find('div','authors').find_all('a')
- self.assertGreater(len(au_a_tags), 1, 'Should be some a tags for authors')
+ au_a_tags = html.find('div', 'authors').find_all('a')
+ self.assertGreater(len(au_a_tags), 1,
+ 'Should be some a tags for authors')
self.assertNotIn('query=The', au_a_tags[0]['href'],
'Collaboration author query should not have "The"')
self.assertEqual(au_a_tags[0].text, 'SuperSuper Collaboration')
-
def test_long_author_colab(self):
id = '1501.05201'
- rv = self.app.get('/abs/'+id)
+ rv = self.app.get('/abs/' + id)
self.assertEqual(rv.status_code, 200)
html = BeautifulSoup(rv.data.decode('utf-8'), 'html.parser')
- auths_elmt = html.find('div','authors')
- self.assertTrue(auths_elmt,'Should authors div element')
+ auths_elmt = html.find('div', 'authors')
+ self.assertTrue(auths_elmt, 'Should authors div element')
a_tags = auths_elmt.find_all('a')
- self.assertEqual(len(a_tags), 2, 'Should be two tags in authors div')
+ self.assertEqual(
+ len(a_tags), 2, 'Should be two tags in authors div')
- colab=a_tags[1]
+ colab = a_tags[1]
- self.assertIsNotNone(colab['href'],' tag in title should have href')
- self.assertEqual(colab['href'], 'https://arxiv.org/search/physics?searchtype=author&query=ILL%2FESS%2FLiU+collaboration')
- self.assertEqual(colab.text, 'ILL/ESS/LiU collaboration for the development of the B10 detector technology in the framework of the CRISP project')
+ self.assertIsNotNone(
+ colab['href'], ' tag in title should have href')
+ self.assertEqual(
+ colab['href'], 'https://arxiv.org/search/physics?searchtype=author&query=ILL%2FESS%2FLiU+collaboration')
+ self.assertEqual(
+ colab.text, 'ILL/ESS/LiU collaboration for the development of the B10 detector technology in the framework of the CRISP project')
-
- @unittest.skip("In current implementation, conflicts with comma test below.")
+ @unittest.skip("In current implementation, conflicts with comma test below.")
def test_space_in_author_list(self):
id = '1210.8438'
- rv = self.app.get('/abs/'+id)
+ rv = self.app.get('/abs/' + id)
self.assertEqual(rv.status_code, 200)
html = BeautifulSoup(rv.data.decode('utf-8'), 'html.parser')
- auths_elmt = html.find('div','authors')
- self.assertTrue(auths_elmt,'Should authors div element')
+ auths_elmt = html.find('div', 'authors')
+ self.assertTrue(auths_elmt, 'Should authors div element')
self.assertIn('Zhe (Rita) Liang,', auths_elmt.text,
'Should be a space after (Rita)')
-
def test_comma_in_author_list(self):
id = '0704.0155'
- rv = self.app.get('/abs/'+id)
+ rv = self.app.get('/abs/' + id)
self.assertEqual(rv.status_code, 200)
html = BeautifulSoup(rv.data.decode('utf-8'), 'html.parser')
auths_elmt = html.find('div', 'authors')
@@ -313,5 +415,34 @@ def test_psi_in_abs(self):
self.assertIn('The phase difference $\phi$, between the superconducting',
abs_elmt.text,
"Expecting uncoverted $\phi$ in html abstract.")
-
+ def test_year(self):
+ rv = self.app.get('/year/astro-ph/09')
+ self.assertEqual(rv.status_code, 200)
+
+ rv = self.app.get('/year/astro-ph/')
+ self.assertEqual( rv.status_code, 200)
+
+ rv = self.app.get('/year/astro-ph')
+ self.assertEqual( rv.status_code, 200)
+
+ rv = self.app.get('/year/astro-ph/09/')
+ self.assertEqual(rv.status_code, 200)
+
+ rv = self.app.get('/year')
+ self.assertEqual( rv.status_code, 404)
+
+ rv = self.app.get('/year/astro-ph/9999')
+ self.assertEqual(rv.status_code, 307, 'Future year should cause temporary redirect')
+
+ rv = self.app.get('/year/fakearchive/01')
+ self.assertNotEqual(rv.status_code, 200)
+ self.assertLess( rv.status_code, 500, 'should not cause a 5XX')
+
+ rv = self.app.get('/year/002/0000')
+ self.assertLess( rv.status_code, 500, 'should not cause a 5XX')
+
+ rv = self.app.get('/year/astro-py/9223372036854775808')
+ self.assertLess( rv.status_code, 500, 'should not cause a 5XX')
+
+
diff --git a/tests/test_categories.py b/tests/test_categories.py
index 00a588420..a05f0b99f 100644
--- a/tests/test_categories.py
+++ b/tests/test_categories.py
@@ -3,7 +3,7 @@
from tests.test_abs_parser import ABS_FILES
from browse.services.document.metadata import AbsMetaSession
-
+from browse.domain.category import Category
class CategoriesTest(unittest.TestCase):
@@ -15,7 +15,7 @@ def test_categories_for_0906_3421v1_cats(self):
['Statistical Mechanics (cond-mat.stat-mech)', 'Mathematical Physics (math-ph)'])
doc = self.absService.get_abs(id)
self.assertIsNotNone(doc)
- assert_that(doc.primary_category.display_str(), equal_to(primary))
+ assert_that(doc.primary_category.display, equal_to(primary))
assert_that(doc.display_secondaries(), equal_to(secondaries))
def test_categories_for_0906_3421_cats(self):
@@ -23,7 +23,7 @@ def test_categories_for_0906_3421_cats(self):
['Statistical Mechanics (cond-mat.stat-mech)', 'Mathematical Physics (math-ph)'])
doc = self.absService.get_abs(id)
self.assertIsNotNone(doc)
- assert_that(doc.primary_category.display_str(), equal_to(primary))
+ assert_that(doc.primary_category.display, equal_to(primary))
assert_that(doc.display_secondaries(), equal_to(secondaries))
def test_categories_for_0704_0129_cats(self):
@@ -31,7 +31,7 @@ def test_categories_for_0704_0129_cats(self):
['Mathematical Physics (math-ph)', 'Analysis of PDEs (math.AP)'])
doc = self.absService.get_abs(id)
self.assertIsNotNone(doc)
- assert_that(doc.primary_category.display_str(), equal_to(primary))
+ assert_that(doc.primary_category.display, equal_to(primary))
assert_that(doc.display_secondaries(), equal_to(secondaries))
def test_categories_for_0704_0914_cats(self):
@@ -39,7 +39,7 @@ def test_categories_for_0704_0914_cats(self):
['Mathematical Physics (math-ph)', 'Optics (physics.optics)'])
doc = self.absService.get_abs(id)
self.assertIsNotNone(doc)
- assert_that(doc.primary_category.display_str(), equal_to(primary))
+ assert_that(doc.primary_category.display, equal_to(primary))
assert_that(doc.display_secondaries(), equal_to(secondaries))
def test_categories_for_0704_0582_cats(self):
@@ -47,7 +47,7 @@ def test_categories_for_0704_0582_cats(self):
'Probability (math.PR)', ['Mathematical Physics (math-ph)'])
doc = self.absService.get_abs(id)
self.assertIsNotNone(doc)
- assert_that(doc.primary_category.display_str(), equal_to(primary))
+ assert_that(doc.primary_category.display, equal_to(primary))
assert_that(doc.display_secondaries(), equal_to(secondaries))
def test_categories_for_0704_0495_cats(self):
@@ -55,7 +55,7 @@ def test_categories_for_0704_0495_cats(self):
['Mathematical Physics (math-ph)'])
doc = self.absService.get_abs(id)
self.assertIsNotNone(doc)
- assert_that(doc.primary_category.display_str(), equal_to(primary))
+ assert_that(doc.primary_category.display, equal_to(primary))
assert_that(doc.display_secondaries(), equal_to(secondaries))
def test_categories_for_0704_0681_cats(self):
@@ -63,7 +63,7 @@ def test_categories_for_0704_0681_cats(self):
['Statistical Mechanics (cond-mat.stat-mech)', 'Instrumentation and Detectors (physics.ins-det)', 'Optics (physics.optics)'])
doc = self.absService.get_abs(id)
self.assertIsNotNone(doc)
- assert_that(doc.primary_category.display_str(), equal_to(primary))
+ assert_that(doc.primary_category.display, equal_to(primary))
assert_that(doc.display_secondaries(), equal_to(secondaries))
def test_categories_for_0704_0761_cats(self):
@@ -71,7 +71,7 @@ def test_categories_for_0704_0761_cats(self):
['Soft Condensed Matter (cond-mat.soft)', 'Chemical Physics (physics.chem-ph)'])
doc = self.absService.get_abs(id)
self.assertIsNotNone(doc)
- assert_that(doc.primary_category.display_str(), equal_to(primary))
+ assert_that(doc.primary_category.display, equal_to(primary))
assert_that(doc.display_secondaries(), equal_to(secondaries))
def test_categories_for_0704_0528_cats(self):
@@ -79,7 +79,7 @@ def test_categories_for_0704_0528_cats(self):
['Information Theory (cs.IT)'])
doc = self.absService.get_abs(id)
self.assertIsNotNone(doc)
- assert_that(doc.primary_category.display_str(), equal_to(primary))
+ assert_that(doc.primary_category.display, equal_to(primary))
assert_that(doc.display_secondaries(), equal_to(secondaries))
def test_categories_for_0704_0869_cats(self):
@@ -87,7 +87,7 @@ def test_categories_for_0704_0869_cats(self):
['Mathematical Physics (math-ph)'])
doc = self.absService.get_abs(id)
self.assertIsNotNone(doc)
- assert_that(doc.primary_category.display_str(), equal_to(primary))
+ assert_that(doc.primary_category.display, equal_to(primary))
assert_that(doc.display_secondaries(), equal_to(secondaries))
def test_categories_for_0704_0796_cats(self):
@@ -95,7 +95,7 @@ def test_categories_for_0704_0796_cats(self):
['Astrophysics (astro-ph)', 'Statistical Mechanics (cond-mat.stat-mech)', 'High Energy Physics - Phenomenology (hep-ph)', 'Mathematical Physics (math-ph)'])
doc = self.absService.get_abs(id)
self.assertIsNotNone(doc)
- assert_that(doc.primary_category.display_str(), equal_to(primary))
+ assert_that(doc.primary_category.display, equal_to(primary))
assert_that(doc.display_secondaries(), equal_to(secondaries))
def test_categories_for_0704_0046_cats(self):
@@ -103,7 +103,7 @@ def test_categories_for_0704_0046_cats(self):
'Quantum Physics (quant-ph)', ['Information Theory (cs.IT)'])
doc = self.absService.get_abs(id)
self.assertIsNotNone(doc)
- assert_that(doc.primary_category.display_str(), equal_to(primary))
+ assert_that(doc.primary_category.display, equal_to(primary))
assert_that(doc.display_secondaries(), equal_to(secondaries))
def test_categories_for_0704_0976_cats(self):
@@ -112,7 +112,7 @@ def test_categories_for_0704_0976_cats(self):
'Chaotic Dynamics (nlin.CD)'])
doc = self.absService.get_abs(id)
self.assertIsNotNone(doc)
- assert_that(doc.primary_category.display_str(), equal_to(primary))
+ assert_that(doc.primary_category.display, equal_to(primary))
assert_that(doc.display_secondaries(), equal_to(secondaries))
def test_categories_for_0704_0041_cats(self):
@@ -120,7 +120,7 @@ def test_categories_for_0704_0041_cats(self):
['Mathematical Physics (math-ph)'])
doc = self.absService.get_abs(id)
self.assertIsNotNone(doc)
- assert_that(doc.primary_category.display_str(), equal_to(primary))
+ assert_that(doc.primary_category.display, equal_to(primary))
assert_that(doc.display_secondaries(), equal_to(secondaries))
def test_categories_for_0704_0918_cats(self):
@@ -128,7 +128,7 @@ def test_categories_for_0704_0918_cats(self):
'Algebraic Geometry (math.AG)', ['Statistics Theory (math.ST)'])
doc = self.absService.get_abs(id)
self.assertIsNotNone(doc)
- assert_that(doc.primary_category.display_str(), equal_to(primary))
+ assert_that(doc.primary_category.display, equal_to(primary))
assert_that(doc.display_secondaries(), equal_to(secondaries))
def test_categories_for_0704_0123_cats(self):
@@ -136,7 +136,7 @@ def test_categories_for_0704_0123_cats(self):
['Other Condensed Matter (cond-mat.other)', 'Optics (physics.optics)'])
doc = self.absService.get_abs(id)
self.assertIsNotNone(doc)
- assert_that(doc.primary_category.display_str(), equal_to(primary))
+ assert_that(doc.primary_category.display, equal_to(primary))
assert_that(doc.display_secondaries(), equal_to(secondaries))
def test_categories_for_0704_0520_cats(self):
@@ -145,7 +145,7 @@ def test_categories_for_0704_0520_cats(self):
'Atomic Physics (physics.atom-ph)'])
doc = self.absService.get_abs(id)
self.assertIsNotNone(doc)
- assert_that(doc.primary_category.display_str(), equal_to(primary))
+ assert_that(doc.primary_category.display, equal_to(primary))
assert_that(doc.display_secondaries(), equal_to(secondaries))
def test_categories_for_0704_0084_cats(self):
@@ -154,7 +154,7 @@ def test_categories_for_0704_0084_cats(self):
'Fluid Dynamics (physics.flu-dyn)'])
doc = self.absService.get_abs(id)
self.assertIsNotNone(doc)
- assert_that(doc.primary_category.display_str(), equal_to(primary))
+ assert_that(doc.primary_category.display, equal_to(primary))
assert_that(doc.display_secondaries(), equal_to(secondaries))
def test_categories_for_0704_0588_cats(self):
@@ -162,7 +162,7 @@ def test_categories_for_0704_0588_cats(self):
'Probability (math.PR)', ['Statistics Theory (math.ST)'])
doc = self.absService.get_abs(id)
self.assertIsNotNone(doc)
- assert_that(doc.primary_category.display_str(), equal_to(primary))
+ assert_that(doc.primary_category.display, equal_to(primary))
assert_that(doc.display_secondaries(), equal_to(secondaries))
def test_categories_for_0704_0687_cats(self):
@@ -170,5 +170,17 @@ def test_categories_for_0704_0687_cats(self):
['Mathematical Physics (math-ph)'])
doc = self.absService.get_abs(id)
self.assertIsNotNone(doc)
- assert_that(doc.primary_category.display_str(), equal_to(primary))
+ assert_that(doc.primary_category.display, equal_to(primary))
assert_that(doc.display_secondaries(), equal_to(secondaries))
+
+
+ def test_non_tax_cats(self):
+ cat = Category('nontx.AL')
+ assert_that(cat.display , equal_to('nontx.AL'))
+
+ # TODO: confirm actual desired behavior
+ # cat = Category('math.XX')
+ # assert_that(cat.display , equal_to('Mathematics (math.XX)'))
+
+ cat = Category('notaxArc')
+ assert_that(cat.display , equal_to('notaxArc'))
diff --git a/tests/test_click_through.py b/tests/test_click_through.py
deleted file mode 100644
index a6a6fd3f5..000000000
--- a/tests/test_click_through.py
+++ /dev/null
@@ -1,15 +0,0 @@
-# from unittest import TestCase
-import unittest
-
-from hypothesis import given
-from hypothesis.strategies import text
-
-from browse.util.clickthrough import create_hash, is_hash_valid
-
-
-class TestClickthrough(unittest.TestCase):
-
- @given(text(), text())
- def test_clickthrough(self, s, v):
- h = create_hash(s, v)
- self.assertTrue(is_hash_valid(s, v, h))
diff --git a/tests/test_cookies.py b/tests/test_cookies.py
new file mode 100644
index 000000000..db1b8b22b
--- /dev/null
+++ b/tests/test_cookies.py
@@ -0,0 +1,37 @@
+"""test cookies"""
+import unittest
+
+#from bs4 import BeautifulSoup
+
+from app import app
+
+
+class CookiesPageTest(unittest.TestCase):
+
+ def setUp(self):
+ app.testing = True
+ app.config['APPLICATION_ROOT'] = ''
+ self.app = app.test_client()
+
+
+ def test_cookies_with_no_params(self):
+ """Test the cookies page."""
+ rv = self.app.get('/cookies')
+ self.assertEqual(rv.status_code, 200)
+ html = rv.data.decode('utf-8')
+ self.assertIn('Select preferred download format', html)
+ self.assertIn('show additional debugging information', html, 'should have SHOW debugging link')
+
+ def test_cookies_with_debug(self):
+ """Test the cookies page."""
+ rv = self.app.get('/cookies?debug=1')
+ self.assertEqual(rv.status_code, 200)
+ html = rv.data.decode('utf-8')
+ self.assertIn('Select preferred download format', html)
+ self.assertIn('hide debugging information', html, 'should have HIDE debugging link')
+
+ def test_post_to_cookies(self):
+ rv = self.app.post('/cookies/set?debug=1', data={'ps':'pdf'})
+ self.assertEqual(rv.status_code, 302)
+ cookies = map(lambda kv: kv[1], filter(lambda kv : kv[0]=='Set-Cookie', rv.headers.items()))
+ self.assertIn('xxx-ps-defaults=pdf; Path=/', cookies)
diff --git a/tests/test_database_service.py b/tests/test_database_service.py
index 025608bda..f6d72e62f 100644
--- a/tests/test_database_service.py
+++ b/tests/test_database_service.py
@@ -4,9 +4,11 @@
from unittest import mock, TestCase
from unittest.mock import Mock, patch
from sqlalchemy.exc import SQLAlchemyError
-
from sqlalchemy.orm.exc import NoResultFound
+from browse.services.database.models import TrackbackPing
+from browse.domain.identifier import Identifier
+
from tests import grep_f_count, execute_sql_files, path_of_for_test
@@ -82,6 +84,10 @@ def setUpClass(cls) -> None:
execute_sql_files(sql_files, database.db.engine)
database.db.session.commit()
+ """Disable logging to avoid messy output during testing"""
+ import logging
+ logging.disable(logging.WARNING)
+
def test_get_institution_returns_a_label(self) -> None:
"""If IP address matches an institution, a label is returned."""
label = TestBrowseDatabaseService.database_service.get_institution(
@@ -123,7 +129,8 @@ def test_get_institution_returns_a_label(self) -> None:
def test_all_trackback_pings(self) -> None:
"""Test if all trackback pings are counted."""
- doc_sql_file = path_of_for_test('data/db/sql/arXiv_trackback_pings.sql')
+ doc_sql_file = path_of_for_test(
+ 'data/db/sql/arXiv_trackback_pings.sql')
count_from_file = grep_f_count(
doc_sql_file,
@@ -151,14 +158,14 @@ def test_all_trackback_pings(self) -> None:
)
def test_trackback_pings(self) -> None:
- """Test if trackback pings for specific paper are counted."""
+ """Test if trackback pings for a specific paper are counted."""
test_paper_id = '0808.4142'
count_from_db: int = TestBrowseDatabaseService.database_service\
.count_trackback_pings(test_paper_id)
count_from_db_list: int = TestBrowseDatabaseService.database_service\
- .get_trackback_pings(test_paper_id).__len__()
+ .get_paper_trackback_pings(test_paper_id).__len__()
self.assertEqual(
- count_from_db, 8,
+ count_from_db, 9,
f'Correct count of pings returned for paper {test_paper_id}'
)
self.assertEqual(
@@ -166,6 +173,21 @@ def test_trackback_pings(self) -> None:
f'Correct count of pings returned for paper {test_paper_id}'
)
+ def test_recent_trackback_pings(self) -> None:
+ """Test if recent trackbacks can be retrieved."""
+ tbs: List = TestBrowseDatabaseService.database_service.\
+ get_recent_trackback_pings(max_trackbacks=-1)
+ self.assertEqual(len(tbs), 0, 'List should be empty')
+ tbs: List = TestBrowseDatabaseService.database_service.\
+ get_recent_trackback_pings(max_trackbacks=25)
+ self.assertGreater(len(tbs), 0, 'List should be nonempty')
+ for tb in tbs:
+ self.assertIsInstance(tb[0], TrackbackPing)
+ self.assertIsInstance(tb[1], str)
+ self.assertIsInstance(Identifier(
+ arxiv_id=tb[1]), Identifier, 'Value looks like an Identifier')
+ self.assertIsInstance(tb[2], str)
+
def test_sciencewise_ping(self) -> None:
"""Test whether paper with version suffix has a ScienceWISE ping."""
test_paper_id_v = '1605.09669v2'
@@ -209,15 +231,23 @@ def test_get_dblp_authors(self) -> None:
TestBrowseDatabaseService.database_service.get_dblp_authors(
test_paper_id), [])
+ def test_get_document_count(self) -> None:
+ """Test document count function."""
+ self.assertGreater(
+ TestBrowseDatabaseService.database_service.get_document_count(),
+ 0,
+ 'There is at least one document in the DB.'
+ )
+
@mock.patch('browse.services.database.models.db.session.query')
- def test_error_conditions(self, mock_query)->None:
+ def test_error_conditions(self, mock_query) -> None:
mock_query.side_effect = NoResultFound
self.assertEqual(
TestBrowseDatabaseService.database_service.get_institution('10.0.0.1'), None)
self.assertEqual([],
- TestBrowseDatabaseService.database_service.get_all_trackback_pings())
+ TestBrowseDatabaseService.database_service.get_all_trackback_pings())
self.assertListEqual(
- TestBrowseDatabaseService.database_service.get_trackback_pings('0704.0361'), [])
+ TestBrowseDatabaseService.database_service.get_paper_trackback_pings('0704.0361'), [])
self.assertEqual(
TestBrowseDatabaseService.database_service.count_trackback_pings('0704.0361'), 0)
self.assertEqual(
@@ -231,12 +261,18 @@ def test_error_conditions(self, mock_query)->None:
mock_query.side_effect = SQLAlchemyError
self.assertRaises(SQLAlchemyError,
TestBrowseDatabaseService.database_service.get_institution, '10.0.0.1')
- self.assertRaises(SQLAlchemyError, TestBrowseDatabaseService.database_service.get_all_trackback_pings)
- self.assertRaises(SQLAlchemyError, TestBrowseDatabaseService.database_service.get_trackback_pings, 'paperx')
- self.assertRaises(SQLAlchemyError, TestBrowseDatabaseService.database_service.count_all_trackback_pings)
- self.assertRaises(SQLAlchemyError, TestBrowseDatabaseService.database_service.has_sciencewise_ping, 'px')
- self.assertRaises(SQLAlchemyError, TestBrowseDatabaseService.database_service.get_dblp_listing_path, 'px')
- self.assertRaises(SQLAlchemyError, TestBrowseDatabaseService.database_service.get_dblp_authors,'authx')
+ self.assertRaises(
+ SQLAlchemyError, TestBrowseDatabaseService.database_service.get_all_trackback_pings)
+ self.assertRaises(
+ SQLAlchemyError, TestBrowseDatabaseService.database_service.get_paper_trackback_pings, 'paperx')
+ self.assertRaises(
+ SQLAlchemyError, TestBrowseDatabaseService.database_service.count_all_trackback_pings)
+ self.assertRaises(
+ SQLAlchemyError, TestBrowseDatabaseService.database_service.has_sciencewise_ping, 'px')
+ self.assertRaises(
+ SQLAlchemyError, TestBrowseDatabaseService.database_service.get_dblp_listing_path, 'px')
+ self.assertRaises(
+ SQLAlchemyError, TestBrowseDatabaseService.database_service.get_dblp_authors, 'authx')
@classmethod
def tearDownClass(cls) -> None:
diff --git a/tests/test_doi.py b/tests/test_doi.py
index 124106d76..18772845b 100644
--- a/tests/test_doi.py
+++ b/tests/test_doi.py
@@ -13,6 +13,7 @@
from app import app
+@unittest.skip('We will move this test and any required test data to arxiv-base in the near future')
class DoiTest(unittest.TestCase):
def setUp(self):
diff --git a/tests/test_exceptions.py b/tests/test_exceptions.py
index 54186696b..2ddc2e6d0 100644
--- a/tests/test_exceptions.py
+++ b/tests/test_exceptions.py
@@ -19,6 +19,8 @@ def setUp(self):
wlog.disabled = True
self.app = create_web_app()
+ self.app.testing = True
+ self.app.config['APPLICATION_ROOT'] = ''
self.client = self.app.test_client()
def test_404(self):
@@ -54,7 +56,8 @@ def test_500(self, mock_abs):
"""Disable logging to avoid messy output during testing"""
self.app.logger.disabled = True
- response = self.client.get('/abs/1234.5678')
- self.assertEqual(response.status_code,
- status.HTTP_500_INTERNAL_SERVER_ERROR)
- self.assertIn('text/html', response.content_type)
+ with self.assertRaises(AbsException):
+ response = self.client.get('/abs/1234.5678')
+ self.assertEqual(response.status_code,
+ status.HTTP_500_INTERNAL_SERVER_ERROR)
+ self.assertIn('text/html', response.content_type)
diff --git a/tests/test_filters.py b/tests/test_filters.py
index ee38e22e1..31c7d8e00 100644
--- a/tests/test_filters.py
+++ b/tests/test_filters.py
@@ -1,5 +1,4 @@
import unittest
-from hamcrest import *
from functools import partial
from jinja2 import escape, Markup, Environment
@@ -7,229 +6,217 @@
from flask import appcontext_pushed, url_for
from app import app
-from browse.filters import line_feed_to_br, tex_to_utf, entity_to_utf
-from browse.util.id_patterns import do_dois_to_tags, do_dois_id_urls_to_tags,\
- do_id_to_tags
+from arxiv.base.urls import links, urlizer, urlize
+from arxiv.base.filters import abstract_lf_to_br, f_tex2utf
+from browse.filters import entity_to_utf
+@unittest.skip("These test features implemented in arxiv-base so move these tests to arxiv-base")
+class Jinja_Custom_Filters_Test(unittest.TestCase):
+ """Browse jinja filter tests."""
-def _id_to_url(id: str):
- return url_for('browse.abstract', arxiv_id=id)
-
-def arxiv_urlize(txt):
- return do_dois_id_urls_to_tags(_id_to_url,None,txt)
-
-def doi_urls(fn, txt):
- return do_dois_to_tags(fn, txt)
-
-def arxiv_id_urls(txt):
- return do_id_to_tags(_id_to_url, txt)
-
-class Jinja_Custom_Fitlers_Test(unittest.TestCase):
def test_with_jinja(self):
- jenv = Environment(autoescape=True)
- jenv.filters['doi_urls'] = partial(doi_urls, lambda x: x)
- assert_that(
- jenv.from_string(
- '{{"something 10.1103/PhysRevD.76.013009 or other"|doi_urls}}'
- ).render(),
- equal_to(
- 'something 10.1103/PhysRevD.76.013009 or other'
- ))
+ """Basic urlize DOI filter test."""
+ with app.app_context():
+ jenv = Environment(autoescape=True)
+ jenv.filters['urlize'] = urlizer(
+ ['doi']
+ )
+ self.assertEqual(
+ jenv.from_string(
+ '{{"something 10.1103/PhysRevD.76.013009 or other"|urlize}}'
+ ).render(),
+ 'something <a class="link-https link-external" data-doi="10.1103/PhysRevD.76.013009" href="https://arxiv.org/ct?url=https%3A%2F%2Fdx.doi.org%2F10.1103%2FPhysRevD.76.013009&v=d0670bbf" rel="external noopener nofollow">10.1103/PhysRevD.76.013009</a> or other'
+ )
def test_with_jinja_escapes(self):
- jenv = Environment(autoescape=True)
- jenv.filters['doi_urls'] = partial(doi_urls, lambda x: x)
- jenv.filters['arxiv_urlize'] = arxiv_urlize
- assert_that(
- jenv.from_string(
- '{{"something 10.1103/PhysRevD.76.013009 or other"|doi_urls}}'
- ).render(),
- equal_to(
- 'something 10.1103/PhysRevD.76.013009 or other'
- ))
-
- assert_that(
- jenv.from_string(
- '{{" something 10.1103/PhysRevD.76.013009"|arxiv_urlize}}'
- ).render(),
- equal_to(
- '<script>bad junk</script> something 10.1103/PhysRevD.76.013009'
- ))
+ """Test the tex2utf filter with jinja escapes."""
+ with app.app_context():
+ jenv = Environment(autoescape=True)
+ jenv.filters['urlize'] = urlizer(
+ ['arxiv_id', 'doi']
+ )
+
+ # TODO: urlize doesn't seem to return a Markup object?
+ self.assertEqual(
+ jenv.from_string(
+ '{{"something 10.1103/PhysRevD.76.013009 or other"|urlize}}'
+ ).render(),
+ 'something <a class="link-https link-external" data-doi="10.1103/PhysRevD.76.013009" href="https://arxiv.org/ct?url=https%3A%2F%2Fdx.doi.org%2F10.1103%2FPhysRevD.76.013009&v=d0670bbf" rel="external noopener nofollow">10.1103/PhysRevD.76.013009</a> or other'
+ )
+
+ self.assertEqual(
+ jenv.from_string(
+ '{{" something 10.1103/PhysRevD.76.013009"|urlize}}'
+ ).render(),
+ '<script>bad junk</script> something <a class="link-https link-external" data-doi="10.1103/PhysRevD.76.013009" href="https://arxiv.org/ct?url=https%3A%2F%2Fdx.doi.org%2F10.1103%2FPhysRevD.76.013009&v=d0670bbf" rel="external noopener nofollow">10.1103/PhysRevD.76.013009</a>'
+ )
def test_doi_filter(self):
- doi_fn = partial(doi_urls, lambda x: x)
-
- s = ''
- self.assertEqual(doi_fn(s), s)
-
- s = 'some test string 23$6#$5<>&456 http://google.com/notadoi'
- assert_that(doi_fn(s), equal_to(escape(s)))
-
- doi = '10.1103/PhysRevD.76.013009'
- doiurl = doi_fn(doi)
- self.assertRegex(doiurl, r'^10.1103/PhysRevD.76.013009
div should be escaped
'
- ex_txt = escape(user_entered_txt).__html__()
- assert_that(
- arxiv_id_urls(ax_id + user_entered_txt),
- equal_to(
- f'hep-th/9901002{ex_txt}'
- ),
+ ex_txt = Markup(user_entered_txt)
+ self.assertEqual(
+ urlize(ax_id + user_entered_txt),
+ f'hep-th/9901002{ex_txt}',
'Dealing with user entered text with html that should be escaped for safety'
)
jinja_escaped_txt = Markup(
' div should already be escaped by jinja2
')
- assert_that(
- arxiv_id_urls(ax_id + jinja_escaped_txt),
- equal_to(
- f'hep-th/9901002{jinja_escaped_txt}'
- ), 'Dealing with text that has been escaped by Jinja2 already')
+ self.assertEqual(
+ urlize(ax_id + jinja_escaped_txt),
+ f'hep-th/9901002{jinja_escaped_txt}',
+ 'Dealing with text that has been escaped by Jinja2 already')
def test_arxiv_id_jinja_escapes(self):
h = 'sosmooth.org'
@@ -237,224 +224,209 @@ def test_arxiv_id_jinja_escapes(self):
with app.app_context():
jenv = Environment(autoescape=True)
- jenv.filters['arxiv_id_urls'] = arxiv_id_urls
- jenv.filters['arxiv_urlize'] = arxiv_urlize
+ jenv.filters['urlize'] = jenv.filters['urlize'] = urlizer(
+ ['arxiv_id', 'doi', 'url']
+ )
- assert_that(
+ self.assertEqual(
jenv.from_string(
- '{{"something hep-th/9901002 or other"|arxiv_id_urls}}').
+ '{{"something hep-th/9901002 or other"|urlize|safe}}').
render(),
- equal_to(
- f'something hep-th/9901002 or other'
- ))
+ f'something hep-th/9901002 or other'
+ )
- assert_that(
+ self.assertEqual(
jenv.from_string(
- '{{" something 10.1103/PhysRevD.76.013009"|arxiv_urlize}}'
+ '{{" something 10.1103/PhysRevD.76.013009"|urlize|safe}}'
).render(),
- equal_to(
- '<script>bad junk</script> something 10.1103/PhysRevD.76.013009'
- ))
+ ' something 10.1103/PhysRevD.76.013009'
+ )
- assert_that(
+ self.assertEqual(
jenv.from_string(
'{{" http://google.com bla bla '
- 'hep-th/9901002 bla"|arxiv_urlize}}').
+ 'hep-th/9901002 bla"|urlize|safe}}').
render(),
- equal_to(
- '<script>bad junk</script> '
- 'this http URL bla bla '
- f'hep-th/9901002 bla'
- ), 'should not double escape')
+ ' '
+ 'this http URL bla bla '
+ f'hep-th/9901002 bla',
+ 'should not double escape')
def test_line_break(self):
- assert_that(
- line_feed_to_br('blal\n bla'), equal_to('blal\nbla')) + """Test the abstract lf to br tag filter.""" + self.assertEqual(abstract_lf_to_br('blal\n bla'), 'blal\n
bla') - assert_that( - line_feed_to_br('\nblal\n bla'), equal_to('\nblal\n
bla')) + self.assertEqual(abstract_lf_to_br( + '\nblal\n bla'), '\nblal\n
bla') - assert_that( - line_feed_to_br('\n blal\n bla'), equal_to('\n blal\n
bla'), - 'need to not do
on first line') - assert_that( - line_feed_to_br('blal\n\nbla'), equal_to('blal\nbla'), - 'skip blank lines') + self.assertEqual(abstract_lf_to_br('\n blal\n bla'), + '\n blal\n
bla', + 'need to not do
on first line') + self.assertEqual(abstract_lf_to_br('blal\n\nbla'), 'blal\nbla', + 'skip blank lines') def test_line_break_jinja(self): + """Test the abstract lf to br tag filter with urlize on arXiv IDs.""" h = 'sosmooth.org' app.config['SERVER_NAME'] = h with app.app_context(): jenv = Environment(autoescape=True) - jenv.filters['arxiv_urlize'] = arxiv_urlize - jenv.filters['line_break'] = line_feed_to_br + jenv.filters['urlize'] = urlize + jenv.filters['line_break'] = abstract_lf_to_br - assert_that( + self.assertEqual( jenv.from_string( '{{" http://google.com something or \n' '\n' 'no double \\n' ' should have br\n' - 'hep-th/9901002 other"|line_break|arxiv_urlize}}' + 'hep-th/9901002 other"|line_break|urlize|safe}}' ).render(), - equal_to( - '<script>bad junk</script> ' - 'this http URL' - ' something or \n' - 'no double \n' - '
should have br\n' - 'hep-th/9901002 other' - ), + '<script>bad junk</script> ' + 'this http URL' + ' something or \n' + 'no double \n' + '
should have br\n' + 'hep-th/9901002 other', 'line_break and arxiv_id_urls should work together' ) - def test_tex_to_utf(self): + def test_tex2utf(self): + """Test the tex2utf filter.""" h = 'sosmooth.org' app.config['SERVER_NAME'] = h with app.app_context(): jenv = Environment(autoescape=True) - jenv.filters['arxiv_id_urls'] = arxiv_id_urls - jenv.filters['line_break'] = line_feed_to_br - jenv.filters['doi_urls'] = partial(doi_urls, lambda x: x) - jenv.filters['arxiv_urlize'] = arxiv_urlize - jenv.filters['tex_to_utf'] = tex_to_utf + jenv.filters['urlize'] = urlize + jenv.filters['line_break'] = abstract_lf_to_br + jenv.filters['tex2utf'] = f_tex2utf - assert_that( - jenv.from_string('{{""|tex_to_utf|arxiv_id_urls}}').render(), - equal_to('')) + self.assertEqual( + jenv.from_string('{{""|tex2utf|urlize|safe}}').render(), + '' + ) title = jenv.from_string( - '{{"Finite-Size and Finite-Temperature Effects in the Conformally Invariant O(N) Vector Model for 2