From e1529c48fe2dd83c81cc91a09d3b80fdf40e16bb Mon Sep 17 00:00:00 2001 From: Casey Greene Date: Tue, 2 Aug 2016 13:41:56 -0400 Subject: [PATCH 001/216] Initial commit --- README.md | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..e205917 --- /dev/null +++ b/README.md @@ -0,0 +1,2 @@ +# deep-review +Deep Learning, Genomics, and Precision Medicine From c98f9789c6fdc9a7313456fb543fbe44b805907e Mon Sep 17 00:00:00 2001 From: Casey Greene Date: Tue, 2 Aug 2016 13:54:09 -0400 Subject: [PATCH 002/216] draft description of repo --- README.md | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e205917..a451199 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,11 @@ -# deep-review -Deep Learning, Genomics, and Precision Medicine +# Deep Learning, Genomics, and Precision Medicine + +We have the opportunity to write a headline review for *Journal of the Royal Society Interface* on a topic overlapping the computer and life sciences in the area of systems pharmacology + +> A Headline Review is one in a short, targeted series of high-level reviews within a particular topic of a burgeoning research area. We encourage authors to write in a style that opens the door to a broad range of readers working at the physical sciences - life sciences interface. We intend the reviews to address critical developments in an area of cross-disciplinary research and, when possible, to place such research in a broader context. This is not a place for comprehensive literature surveys. +> +> We do encourage you to speculate in an informed way, and to be topical and provocative about the subject without worrying unduly about space, (the provisional target length is 8 -12,000 words). Please think of this as an article which will be a landmark in your area, and will come to be considered as a classic paper of the literature. + +I was recently inspired by [Harold Pimentel's](https://github.com/pimentel) crowdsourced [collection of deep learning papers](https://github.com/pimentel/deep_learning_papers). Instead of having one individual write this, I thought that this invitation provided a wonderful opportunity to take advantage of the wisdom of crowds to bring a team together around this topic. + +This repository provides a home for the paper. We'll operate on a pull request model. Anyone whose contributions meet the ICJME standards of authorship will be included as an author on the manuscript. I can't guarantee that it will be accepted, but I look forward to trying this approach out. From de37b1863b7bb090c20407584ec6db228aa483b4 Mon Sep 17 00:00:00 2001 From: Casey Greene Date: Tue, 2 Aug 2016 17:28:36 -0400 Subject: [PATCH 003/216] Add license, update readme, add how to contribute --- CONTRIBUTING.md | 34 +++++ LICENSE.md | 395 ++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 4 +- 3 files changed, 431 insertions(+), 2 deletions(-) create mode 100644 CONTRIBUTING.md create mode 100644 LICENSE.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..8ff12b9 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,34 @@ +# Contribution guidelines for the deep-review repository + +## Issues + +We'll use issues for discussion of papers, section outlines, and other +structural components of the paper. + +## Pull requests + +Contributions to the article operate on a pull request model. We expect +participants to actively review pull requests. We'd love to have you ask +questions, clarify points, and jump in and edit the text. + +## Authorship + +What qualifies as authorship? We use the [ICJME Guidelines](http://www.icmje.org/recommendations/browse/roles-and-responsibilities/defining-the-role-of-authors-and-contributors.html). +We expect authors to contribute to the overall design by participating in +issues, to contribute to the text by contributing sections and/or revisions +to sections through pull requests. It is important to note that, for authorship, +these should be substantial intellectual contributions. + +## Peer review + +All pull requests will undergo peer review. Participants in this project should +review the changes. They should suggest modifications or, potentially, directly +edit the pull request to make suggested changes. Reviewers can note approval by +commenting ":+1:", "looks good to me", or "approved". As a reviewer, it's +helpful to note the type of review you performed: did you read cited literature, +look over the text in detail, or are you just supporting the concept? + +Before a repository maintainer merges a pull request, there must be at least +one affirmative review. If there is any unaddressed criticism or disapproval, +a repository maintainer will determine how to proceed and may wait for +additional feedback. diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 0000000..2f244ac --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,395 @@ +Attribution 4.0 International + +======================================================================= + +Creative Commons Corporation ("Creative Commons") is not a law firm and +does not provide legal services or legal advice. Distribution of +Creative Commons public licenses does not create a lawyer-client or +other relationship. Creative Commons makes its licenses and related +information available on an "as-is" basis. Creative Commons gives no +warranties regarding its licenses, any material licensed under their +terms and conditions, or any related information. Creative Commons +disclaims all liability for damages resulting from their use to the +fullest extent possible. + +Using Creative Commons Public Licenses + +Creative Commons public licenses provide a standard set of terms and +conditions that creators and other rights holders may use to share +original works of authorship and other material subject to copyright +and certain other rights specified in the public license below. The +following considerations are for informational purposes only, are not +exhaustive, and do not form part of our licenses. + + Considerations for licensors: Our public licenses are + intended for use by those authorized to give the public + permission to use material in ways otherwise restricted by + copyright and certain other rights. Our licenses are + irrevocable. Licensors should read and understand the terms + and conditions of the license they choose before applying it. + Licensors should also secure all rights necessary before + applying our licenses so that the public can reuse the + material as expected. Licensors should clearly mark any + material not subject to the license. This includes other CC- + licensed material, or material used under an exception or + limitation to copyright. More considerations for licensors: + wiki.creativecommons.org/Considerations_for_licensors + + Considerations for the public: By using one of our public + licenses, a licensor grants the public permission to use the + licensed material under specified terms and conditions. If + the licensor's permission is not necessary for any reason--for + example, because of any applicable exception or limitation to + copyright--then that use is not regulated by the license. Our + licenses grant only permissions under copyright and certain + other rights that a licensor has authority to grant. Use of + the licensed material may still be restricted for other + reasons, including because others have copyright or other + rights in the material. A licensor may make special requests, + such as asking that all changes be marked or described. + Although not required by our licenses, you are encouraged to + respect those requests where reasonable. More_considerations + for the public: + wiki.creativecommons.org/Considerations_for_licensees + +======================================================================= + +Creative Commons Attribution 4.0 International Public License + +By exercising the Licensed Rights (defined below), You accept and agree +to be bound by the terms and conditions of this Creative Commons +Attribution 4.0 International Public License ("Public License"). To the +extent this Public License may be interpreted as a contract, You are +granted the Licensed Rights in consideration of Your acceptance of +these terms and conditions, and the Licensor grants You such rights in +consideration of benefits the Licensor receives from making the +Licensed Material available under these terms and conditions. + + +Section 1 -- Definitions. + + a. Adapted Material means material subject to Copyright and Similar + Rights that is derived from or based upon the Licensed Material + and in which the Licensed Material is translated, altered, + arranged, transformed, or otherwise modified in a manner requiring + permission under the Copyright and Similar Rights held by the + Licensor. For purposes of this Public License, where the Licensed + Material is a musical work, performance, or sound recording, + Adapted Material is always produced where the Licensed Material is + synched in timed relation with a moving image. + + b. Adapter's License means the license You apply to Your Copyright + and Similar Rights in Your contributions to Adapted Material in + accordance with the terms and conditions of this Public License. + + c. Copyright and Similar Rights means copyright and/or similar rights + closely related to copyright including, without limitation, + performance, broadcast, sound recording, and Sui Generis Database + Rights, without regard to how the rights are labeled or + categorized. For purposes of this Public License, the rights + specified in Section 2(b)(1)-(2) are not Copyright and Similar + Rights. + + d. Effective Technological Measures means those measures that, in the + absence of proper authority, may not be circumvented under laws + fulfilling obligations under Article 11 of the WIPO Copyright + Treaty adopted on December 20, 1996, and/or similar international + agreements. + + e. Exceptions and Limitations means fair use, fair dealing, and/or + any other exception or limitation to Copyright and Similar Rights + that applies to Your use of the Licensed Material. + + f. Licensed Material means the artistic or literary work, database, + or other material to which the Licensor applied this Public + License. + + g. Licensed Rights means the rights granted to You subject to the + terms and conditions of this Public License, which are limited to + all Copyright and Similar Rights that apply to Your use of the + Licensed Material and that the Licensor has authority to license. + + h. Licensor means the individual(s) or entity(ies) granting rights + under this Public License. + + i. Share means to provide material to the public by any means or + process that requires permission under the Licensed Rights, such + as reproduction, public display, public performance, distribution, + dissemination, communication, or importation, and to make material + available to the public including in ways that members of the + public may access the material from a place and at a time + individually chosen by them. + + j. Sui Generis Database Rights means rights other than copyright + resulting from Directive 96/9/EC of the European Parliament and of + the Council of 11 March 1996 on the legal protection of databases, + as amended and/or succeeded, as well as other essentially + equivalent rights anywhere in the world. + + k. You means the individual or entity exercising the Licensed Rights + under this Public License. Your has a corresponding meaning. + + +Section 2 -- Scope. + + a. License grant. + + 1. Subject to the terms and conditions of this Public License, + the Licensor hereby grants You a worldwide, royalty-free, + non-sublicensable, non-exclusive, irrevocable license to + exercise the Licensed Rights in the Licensed Material to: + + a. reproduce and Share the Licensed Material, in whole or + in part; and + + b. produce, reproduce, and Share Adapted Material. + + 2. Exceptions and Limitations. For the avoidance of doubt, where + Exceptions and Limitations apply to Your use, this Public + License does not apply, and You do not need to comply with + its terms and conditions. + + 3. Term. The term of this Public License is specified in Section + 6(a). + + 4. Media and formats; technical modifications allowed. The + Licensor authorizes You to exercise the Licensed Rights in + all media and formats whether now known or hereafter created, + and to make technical modifications necessary to do so. The + Licensor waives and/or agrees not to assert any right or + authority to forbid You from making technical modifications + necessary to exercise the Licensed Rights, including + technical modifications necessary to circumvent Effective + Technological Measures. For purposes of this Public License, + simply making modifications authorized by this Section 2(a) + (4) never produces Adapted Material. + + 5. Downstream recipients. + + a. Offer from the Licensor -- Licensed Material. Every + recipient of the Licensed Material automatically + receives an offer from the Licensor to exercise the + Licensed Rights under the terms and conditions of this + Public License. + + b. No downstream restrictions. You may not offer or impose + any additional or different terms or conditions on, or + apply any Effective Technological Measures to, the + Licensed Material if doing so restricts exercise of the + Licensed Rights by any recipient of the Licensed + Material. + + 6. No endorsement. Nothing in this Public License constitutes or + may be construed as permission to assert or imply that You + are, or that Your use of the Licensed Material is, connected + with, or sponsored, endorsed, or granted official status by, + the Licensor or others designated to receive attribution as + provided in Section 3(a)(1)(A)(i). + + b. Other rights. + + 1. Moral rights, such as the right of integrity, are not + licensed under this Public License, nor are publicity, + privacy, and/or other similar personality rights; however, to + the extent possible, the Licensor waives and/or agrees not to + assert any such rights held by the Licensor to the limited + extent necessary to allow You to exercise the Licensed + Rights, but not otherwise. + + 2. Patent and trademark rights are not licensed under this + Public License. + + 3. To the extent possible, the Licensor waives any right to + collect royalties from You for the exercise of the Licensed + Rights, whether directly or through a collecting society + under any voluntary or waivable statutory or compulsory + licensing scheme. In all other cases the Licensor expressly + reserves any right to collect such royalties. + + +Section 3 -- License Conditions. + +Your exercise of the Licensed Rights is expressly made subject to the +following conditions. + + a. Attribution. + + 1. If You Share the Licensed Material (including in modified + form), You must: + + a. retain the following if it is supplied by the Licensor + with the Licensed Material: + + i. identification of the creator(s) of the Licensed + Material and any others designated to receive + attribution, in any reasonable manner requested by + the Licensor (including by pseudonym if + designated); + + ii. a copyright notice; + + iii. a notice that refers to this Public License; + + iv. a notice that refers to the disclaimer of + warranties; + + v. a URI or hyperlink to the Licensed Material to the + extent reasonably practicable; + + b. indicate if You modified the Licensed Material and + retain an indication of any previous modifications; and + + c. indicate the Licensed Material is licensed under this + Public License, and include the text of, or the URI or + hyperlink to, this Public License. + + 2. You may satisfy the conditions in Section 3(a)(1) in any + reasonable manner based on the medium, means, and context in + which You Share the Licensed Material. For example, it may be + reasonable to satisfy the conditions by providing a URI or + hyperlink to a resource that includes the required + information. + + 3. If requested by the Licensor, You must remove any of the + information required by Section 3(a)(1)(A) to the extent + reasonably practicable. + + 4. If You Share Adapted Material You produce, the Adapter's + License You apply must not prevent recipients of the Adapted + Material from complying with this Public License. + + +Section 4 -- Sui Generis Database Rights. + +Where the Licensed Rights include Sui Generis Database Rights that +apply to Your use of the Licensed Material: + + a. for the avoidance of doubt, Section 2(a)(1) grants You the right + to extract, reuse, reproduce, and Share all or a substantial + portion of the contents of the database; + + b. if You include all or a substantial portion of the database + contents in a database in which You have Sui Generis Database + Rights, then the database in which You have Sui Generis Database + Rights (but not its individual contents) is Adapted Material; and + + c. You must comply with the conditions in Section 3(a) if You Share + all or a substantial portion of the contents of the database. + +For the avoidance of doubt, this Section 4 supplements and does not +replace Your obligations under this Public License where the Licensed +Rights include other Copyright and Similar Rights. + + +Section 5 -- Disclaimer of Warranties and Limitation of Liability. + + a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE + EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS + AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF + ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS, + IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION, + WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR + PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS, + ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT + KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT + ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU. + + b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE + TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION, + NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT, + INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES, + COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR + USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN + ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR + DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR + IN PART, THIS LIMITATION MAY NOT APPLY TO YOU. + + c. The disclaimer of warranties and limitation of liability provided + above shall be interpreted in a manner that, to the extent + possible, most closely approximates an absolute disclaimer and + waiver of all liability. + + +Section 6 -- Term and Termination. + + a. This Public License applies for the term of the Copyright and + Similar Rights licensed here. However, if You fail to comply with + this Public License, then Your rights under this Public License + terminate automatically. + + b. Where Your right to use the Licensed Material has terminated under + Section 6(a), it reinstates: + + 1. automatically as of the date the violation is cured, provided + it is cured within 30 days of Your discovery of the + violation; or + + 2. upon express reinstatement by the Licensor. + + For the avoidance of doubt, this Section 6(b) does not affect any + right the Licensor may have to seek remedies for Your violations + of this Public License. + + c. For the avoidance of doubt, the Licensor may also offer the + Licensed Material under separate terms or conditions or stop + distributing the Licensed Material at any time; however, doing so + will not terminate this Public License. + + d. Sections 1, 5, 6, 7, and 8 survive termination of this Public + License. + + +Section 7 -- Other Terms and Conditions. + + a. The Licensor shall not be bound by any additional or different + terms or conditions communicated by You unless expressly agreed. + + b. Any arrangements, understandings, or agreements regarding the + Licensed Material not stated herein are separate from and + independent of the terms and conditions of this Public License. + + +Section 8 -- Interpretation. + + a. For the avoidance of doubt, this Public License does not, and + shall not be interpreted to, reduce, limit, restrict, or impose + conditions on any use of the Licensed Material that could lawfully + be made without permission under this Public License. + + b. To the extent possible, if any provision of this Public License is + deemed unenforceable, it shall be automatically reformed to the + minimum extent necessary to make it enforceable. If the provision + cannot be reformed, it shall be severed from this Public License + without affecting the enforceability of the remaining terms and + conditions. + + c. No term or condition of this Public License will be waived and no + failure to comply consented to unless expressly agreed to by the + Licensor. + + d. Nothing in this Public License constitutes or may be interpreted + as a limitation upon, or waiver of, any privileges and immunities + that apply to the Licensor or You, including from the legal + processes of any jurisdiction or authority. + + +======================================================================= + +Creative Commons is not a party to its public +licenses. Notwithstanding, Creative Commons may elect to apply one of +its public licenses to material it publishes and in those instances +will be considered the “Licensor.” The text of the Creative Commons +public licenses is dedicated to the public domain under the CC0 Public +Domain Dedication. Except for the limited purpose of indicating that +material is shared under a Creative Commons public license or as +otherwise permitted by the Creative Commons policies published at +creativecommons.org/policies, Creative Commons does not authorize the +use of the trademark "Creative Commons" or any other trademark or logo +of Creative Commons without its prior written consent including, +without limitation, in connection with any unauthorized modifications +to any of its public licenses or any other arrangements, +understandings, or agreements concerning use of licensed material. For +the avoidance of doubt, this paragraph does not form part of the +public licenses. + +Creative Commons may be contacted at creativecommons.org. diff --git a/README.md b/README.md index a451199..4557cdf 100644 --- a/README.md +++ b/README.md @@ -3,9 +3,9 @@ We have the opportunity to write a headline review for *Journal of the Royal Society Interface* on a topic overlapping the computer and life sciences in the area of systems pharmacology > A Headline Review is one in a short, targeted series of high-level reviews within a particular topic of a burgeoning research area. We encourage authors to write in a style that opens the door to a broad range of readers working at the physical sciences - life sciences interface. We intend the reviews to address critical developments in an area of cross-disciplinary research and, when possible, to place such research in a broader context. This is not a place for comprehensive literature surveys. -> +> > We do encourage you to speculate in an informed way, and to be topical and provocative about the subject without worrying unduly about space, (the provisional target length is 8 -12,000 words). Please think of this as an article which will be a landmark in your area, and will come to be considered as a classic paper of the literature. -I was recently inspired by [Harold Pimentel's](https://github.com/pimentel) crowdsourced [collection of deep learning papers](https://github.com/pimentel/deep_learning_papers). Instead of having one individual write this, I thought that this invitation provided a wonderful opportunity to take advantage of the wisdom of crowds to bring a team together around this topic. +I was recently inspired by [Harold Pimentel's](https://github.com/pimentel) crowd-sourced [collection of deep learning papers](https://github.com/pimentel/deep_learning_papers). Instead of having one individual write this, I thought that this invitation provided a wonderful opportunity to take advantage of the wisdom of crowds to bring a team together around this topic. This repository provides a home for the paper. We'll operate on a pull request model. Anyone whose contributions meet the ICJME standards of authorship will be included as an author on the manuscript. I can't guarantee that it will be accepted, but I look forward to trying this approach out. From 3855f39ad1298ca0467cfda9b359859e702ba6e7 Mon Sep 17 00:00:00 2001 From: Casey Greene Date: Wed, 3 Aug 2016 13:55:24 -0400 Subject: [PATCH 004/216] add 'manuscript needs to be ready by' date --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 4557cdf..98a0f32 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Deep Learning, Genomics, and Precision Medicine -We have the opportunity to write a headline review for *Journal of the Royal Society Interface* on a topic overlapping the computer and life sciences in the area of systems pharmacology +We have the opportunity to write a headline review for *Journal of the Royal Society Interface* on a topic overlapping the computer and life sciences in the area of systems pharmacology. Our aim is to have the paper ready to submit by *November 1, 2016*. > A Headline Review is one in a short, targeted series of high-level reviews within a particular topic of a burgeoning research area. We encourage authors to write in a style that opens the door to a broad range of readers working at the physical sciences - life sciences interface. We intend the reviews to address critical developments in an area of cross-disciplinary research and, when possible, to place such research in a broader context. This is not a place for comprehensive literature surveys. > @@ -9,3 +9,4 @@ We have the opportunity to write a headline review for *Journal of the Royal Soc I was recently inspired by [Harold Pimentel's](https://github.com/pimentel) crowd-sourced [collection of deep learning papers](https://github.com/pimentel/deep_learning_papers). Instead of having one individual write this, I thought that this invitation provided a wonderful opportunity to take advantage of the wisdom of crowds to bring a team together around this topic. This repository provides a home for the paper. We'll operate on a pull request model. Anyone whose contributions meet the ICJME standards of authorship will be included as an author on the manuscript. I can't guarantee that it will be accepted, but I look forward to trying this approach out. + From b2059b54aacd4e82b5ecd110649340a442e3b6fc Mon Sep 17 00:00:00 2001 From: Casey Greene Date: Fri, 5 Aug 2016 10:34:35 -0400 Subject: [PATCH 005/216] fix codeclimate issues that were identified --- LICENSE.md => LICENSE | 0 README.md | 30 +++++++++++++++++++++++++----- 2 files changed, 25 insertions(+), 5 deletions(-) rename LICENSE.md => LICENSE (100%) diff --git a/LICENSE.md b/LICENSE similarity index 100% rename from LICENSE.md rename to LICENSE diff --git a/README.md b/README.md index 4557cdf..db6a44f 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,31 @@ # Deep Learning, Genomics, and Precision Medicine -We have the opportunity to write a headline review for *Journal of the Royal Society Interface* on a topic overlapping the computer and life sciences in the area of systems pharmacology +We have the opportunity to write a headline review for *Journal of the Royal +Society Interface* on a topic overlapping the computer and life sciences in the +area of systems pharmacology -> A Headline Review is one in a short, targeted series of high-level reviews within a particular topic of a burgeoning research area. We encourage authors to write in a style that opens the door to a broad range of readers working at the physical sciences - life sciences interface. We intend the reviews to address critical developments in an area of cross-disciplinary research and, when possible, to place such research in a broader context. This is not a place for comprehensive literature surveys. +> A Headline Review is one in a short, targeted series of high-level reviews +within a particular topic of a burgeoning research area. We encourage authors to +write in a style that opens the door to a broad range of readers working at the +physical sciences - life sciences interface. We intend the reviews to address +critical developments in an area of cross-disciplinary research and, when +possible, to place such research in a broader context. This is not a place for +comprehensive literature surveys. > -> We do encourage you to speculate in an informed way, and to be topical and provocative about the subject without worrying unduly about space, (the provisional target length is 8 -12,000 words). Please think of this as an article which will be a landmark in your area, and will come to be considered as a classic paper of the literature. +> We do encourage you to speculate in an informed way, and to be topical and +provocative about the subject without worrying unduly about space, (the +provisional target length is 8 -12,000 words). Please think of this as an +article which will be a landmark in your area, and will come to be considered as +a classic paper of the literature. -I was recently inspired by [Harold Pimentel's](https://github.com/pimentel) crowd-sourced [collection of deep learning papers](https://github.com/pimentel/deep_learning_papers). Instead of having one individual write this, I thought that this invitation provided a wonderful opportunity to take advantage of the wisdom of crowds to bring a team together around this topic. +I was recently inspired by [Harold Pimentel's](https://github.com/pimentel) +crowd-sourced [collection of deep learning +papers](https://github.com/pimentel/deep_learning_papers). Instead of having one +individual write this, I thought that this invitation provided a wonderful +opportunity to take advantage of the wisdom of crowds to bring a team together +around this topic. -This repository provides a home for the paper. We'll operate on a pull request model. Anyone whose contributions meet the ICJME standards of authorship will be included as an author on the manuscript. I can't guarantee that it will be accepted, but I look forward to trying this approach out. +This repository provides a home for the paper. We'll operate on a pull request +model. Anyone whose contributions meet the ICJME standards of authorship will be +included as an author on the manuscript. I can't guarantee that it will be +accepted, but I look forward to trying this approach out. From d3dcff6a8058c2413dc2255a2e43b89ee7060e58 Mon Sep 17 00:00:00 2001 From: Casey Greene Date: Fri, 5 Aug 2016 10:45:10 -0400 Subject: [PATCH 006/216] add code climate to readme, test config --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 2a5e971..5c4dea2 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # Deep Learning, Genomics, and Precision Medicine +[![Code Climate](https://codeclimate.com/github/greenelab/deep-review/badges/gpa.svg)](https://codeclimate.com/github/greenelab/deep-review) + We have the opportunity to write a headline review for *Journal of the Royal Society Interface* on a topic overlapping the computer and life sciences in the area of systems pharmacology. Our aim is to have the paper ready to submit by From 6160d00f7a08ae5b047586fce8099f4b73dccf20 Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Tue, 9 Aug 2016 11:00:12 -0400 Subject: [PATCH 007/216] Add markdown and citation information Related to comment at https://github.com/greenelab/deep-review/issues/2#issuecomment-238332668 --- CONTRIBUTING.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8ff12b9..ed46143 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -32,3 +32,21 @@ Before a repository maintainer merges a pull request, there must be at least one affirmative review. If there is any unaddressed criticism or disapproval, a repository maintainer will determine how to proceed and may wait for additional feedback. + +## Markdown + +The paper will be written using markdown. Markdown files use the `.md` extension. +Check out the [CommarkMark Help](http://commonmark.org/help/) page for an introduction +to formatting options provided by markdown. In addition, to standard markdown features, +we support [markdown tables](https://help.github.com/articles/organizing-information-with-tables/ +"GitHub Help: Organizing information with tables") and a custom citation syntax. Check out +[Tables Generator](http://www.tablesgenerator.com/markdown_tables) for creating markdown tables. + +The custom citation guidelines are as follows: + +1. Always use a DOI for the version of record if available. Cite DOIs like `[@doi:10.15363/thinklab.4]`. +2. If the version of record doesn't have a DOI but does have a PubMed ID, cite like `[@pmid:26158728]`. +3. If the article is an _arXiv_ preprint, cite like `[@arxiv:1508.06576]`. +4. If the article has none of the above, please file an issue. + +You cite multiple items at once like `[@doi:10.15363/thinklab.4 @pmid:26158728 @arxiv:1508.06576]`. From 2b8425ffbea568f38bed1581cfb03a78a8918d25 Mon Sep 17 00:00:00 2001 From: Casey Greene Date: Thu, 25 Aug 2016 08:30:39 -0400 Subject: [PATCH 008/216] add initial status update to README --- README.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/README.md b/README.md index 5c4dea2..0ad3583 100644 --- a/README.md +++ b/README.md @@ -32,3 +32,28 @@ This repository provides a home for the paper. We'll operate on a pull request model. Anyone whose contributions meet the ICJME standards of authorship will be included as an author on the manuscript. I can't guarantee that it will be accepted, but I look forward to trying this approach out. + +## Status Report on 8/25 + +Over the first three weeks of this project, we've developed an initial guiding +question; collaboratively read, summarized, and discussed existing literature +through github issues; and we're now refining our guiding question. If you want +to begin to contribute to this review now, there are a few steps that you may +want to take to get up to speed quickly. + +1. Read through issue [#2](https://github.com/greenelab/deep-review/issues/2). + This will give an idea of what our perspective was as we were starting out + and planning to read papers. +2. Peruse some of the papers that the group has already reviewed, and take a + look at the review. Fill in gaps that you see in the short summary/discussion + of the paper. +3. Choose some papers in an area that you care about, review them, and summarize + them. +4. Dive in to participation in + [#88](https://github.com/greenelab/deep-review/issues/88) and help us to + further refine the specific question that we're going to deal with in this + review. + +In about a week, we plan to move into the phase where we start to vigorously +discuss this question for each area that the review will cover (currently also +being discussed in [#88](https://github.com/greenelab/deep-review/issues/88)). From 3e2be5a32fbc9e2fd8a83fa98856b66c23486f3f Mon Sep 17 00:00:00 2001 From: Casey Greene Date: Thu, 25 Aug 2016 09:18:34 -0400 Subject: [PATCH 009/216] incorporate suggestions from @gwaygenomics --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0ad3583..6ab2f82 100644 --- a/README.md +++ b/README.md @@ -55,5 +55,6 @@ want to take to get up to speed quickly. review. In about a week, we plan to move into the phase where we start to vigorously -discuss this question for each area that the review will cover (currently also -being discussed in [#88](https://github.com/greenelab/deep-review/issues/88)). +argue about the answer to the question that we coalesce on with +[#88](https://github.com/greenelab/deep-review/issues/88) for each area that the +review will cover. From e1b95881a4b261152fbcdf14c5e50e432d6ebd2d Mon Sep 17 00:00:00 2001 From: Casey Greene Date: Thu, 25 Aug 2016 09:20:07 -0400 Subject: [PATCH 010/216] more tuning of get involved list --- README.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 6ab2f82..e1ac64d 100644 --- a/README.md +++ b/README.md @@ -49,10 +49,9 @@ want to take to get up to speed quickly. of the paper. 3. Choose some papers in an area that you care about, review them, and summarize them. -4. Dive in to participation in - [#88](https://github.com/greenelab/deep-review/issues/88) and help us to - further refine the specific question that we're going to deal with in this - review. +4. Dive into [#88](https://github.com/greenelab/deep-review/issues/88) and help + us to further refine the specific question that we're going to deal with in + this review. In about a week, we plan to move into the phase where we start to vigorously argue about the answer to the question that we coalesce on with From 9bd1c345cde3812ff6091f9b65d8cc6af8feb43b Mon Sep 17 00:00:00 2001 From: Casey Greene Date: Thu, 13 Oct 2016 09:49:34 -0400 Subject: [PATCH 011/216] add initial stub files --- build.sh | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 build.sh diff --git a/build.sh b/build.sh new file mode 100644 index 0000000..2dbf391 --- /dev/null +++ b/build.sh @@ -0,0 +1,2 @@ + +pandoc -f markdown -o review.pdf sections/*.md From e0bb1fc23a23482fddcaf1937780331b14f80e73 Mon Sep 17 00:00:00 2001 From: Casey Greene Date: Thu, 13 Oct 2016 10:17:01 -0400 Subject: [PATCH 012/216] fix up pandoc stubs --- build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sh b/build.sh index 2dbf391..8e79993 100644 --- a/build.sh +++ b/build.sh @@ -1,2 +1,2 @@ -pandoc -f markdown -o review.pdf sections/*.md +pandoc --verbose --from=markdown --output=review.pdf sections/*.md From bb55bfe254be4dc4281a24e6448b5a5dea1097ae Mon Sep 17 00:00:00 2001 From: Casey Greene Date: Thu, 20 Oct 2016 10:51:00 -0400 Subject: [PATCH 013/216] reality has struck! --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e1ac64d..298107e 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ We have the opportunity to write a headline review for *Journal of the Royal Society Interface* on a topic overlapping the computer and life sciences in the area of systems pharmacology. Our aim is to have the paper ready to submit by -*November 1, 2016*. +*December 1, 2016*. > A Headline Review is one in a short, targeted series of high-level reviews within a particular topic of a burgeoning research area. We encourage authors to From 69dfd7a41cabcddccd6c17bf72a61f1453d60712 Mon Sep 17 00:00:00 2001 From: Anthony Gitter Date: Wed, 26 Oct 2016 16:49:29 -0500 Subject: [PATCH 014/216] 2016-10-26 status report --- README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/README.md b/README.md index 298107e..0b8995d 100644 --- a/README.md +++ b/README.md @@ -57,3 +57,17 @@ In about a week, we plan to move into the phase where we start to vigorously argue about the answer to the question that we coalesce on with [#88](https://github.com/greenelab/deep-review/issues/88) for each area that the review will cover. + +## Status Report on 10/26 + +We are now actively outlining the review sections and will begin writing +soon. The goal is to have a complete draft in about a month. The action items +from the 8/25 status report are still applicable. In addition, you can: + +1. Sign up to write in [#116](https://github.com/greenelab/deep-review/issues/116) + and share which sections you are most interested in. We are in need of + experts in biomedical imaging applications in particular. +2. Review the stubs in the `sections` subdirectory and respond to the prompts + with a pull request. +3. Outline sections that do not have stubs with a pull request and discuss + them with other co-authors in the pull request comments. From a660556413f66843be9a37c33cb5953c8318766b Mon Sep 17 00:00:00 2001 From: Anthony Gitter Date: Thu, 27 Oct 2016 07:16:46 -0500 Subject: [PATCH 015/216] Rearrange status update order --- README.md | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 0b8995d..f820d9e 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,20 @@ model. Anyone whose contributions meet the ICJME standards of authorship will be included as an author on the manuscript. I can't guarantee that it will be accepted, but I look forward to trying this approach out. +## Status Report on 10/26 + +We are now actively outlining the review sections and will begin writing +soon. The goal is to have a complete draft in about a month. The action items +from the 8/25 status report below are still applicable. In addition, you can: + +1. Sign up to write in [#116](https://github.com/greenelab/deep-review/issues/116) + and share which sections you are most interested in. We are in need of + experts in biomedical imaging applications in particular. +2. Review the stubs in the `sections` subdirectory and respond to the prompts + with a pull request. +3. Outline sections that do not have stubs with a pull request and discuss + them with other co-authors in the pull request comments. + ## Status Report on 8/25 Over the first three weeks of this project, we've developed an initial guiding @@ -58,16 +72,3 @@ argue about the answer to the question that we coalesce on with [#88](https://github.com/greenelab/deep-review/issues/88) for each area that the review will cover. -## Status Report on 10/26 - -We are now actively outlining the review sections and will begin writing -soon. The goal is to have a complete draft in about a month. The action items -from the 8/25 status report are still applicable. In addition, you can: - -1. Sign up to write in [#116](https://github.com/greenelab/deep-review/issues/116) - and share which sections you are most interested in. We are in need of - experts in biomedical imaging applications in particular. -2. Review the stubs in the `sections` subdirectory and respond to the prompts - with a pull request. -3. Outline sections that do not have stubs with a pull request and discuss - them with other co-authors in the pull request comments. From 80807fde8a8340f082976682c28fc88b6762948d Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Sun, 30 Oct 2016 09:44:35 -0400 Subject: [PATCH 016/216] Enable tag citations (#123) * Enable tag citations See https://github.com/greenelab/deep-review/issues/2#issuecomment-256496372. * Reflow text to satisfy codeclimate --- CONTRIBUTING.md | 41 ++++++++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ed46143..0fea540 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -35,18 +35,29 @@ additional feedback. ## Markdown -The paper will be written using markdown. Markdown files use the `.md` extension. -Check out the [CommarkMark Help](http://commonmark.org/help/) page for an introduction -to formatting options provided by markdown. In addition, to standard markdown features, -we support [markdown tables](https://help.github.com/articles/organizing-information-with-tables/ -"GitHub Help: Organizing information with tables") and a custom citation syntax. Check out -[Tables Generator](http://www.tablesgenerator.com/markdown_tables) for creating markdown tables. - -The custom citation guidelines are as follows: - -1. Always use a DOI for the version of record if available. Cite DOIs like `[@doi:10.15363/thinklab.4]`. -2. If the version of record doesn't have a DOI but does have a PubMed ID, cite like `[@pmid:26158728]`. -3. If the article is an _arXiv_ preprint, cite like `[@arxiv:1508.06576]`. -4. If the article has none of the above, please file an issue. - -You cite multiple items at once like `[@doi:10.15363/thinklab.4 @pmid:26158728 @arxiv:1508.06576]`. +The paper will be written using markdown. Markdown files use the `.md` +extension. Check out the [CommarkMark Help](http://commonmark.org/help/) page +for an introduction to formatting options provided by markdown. In addition, to +standard markdown features, we support [markdown +tables](https://help.github.com/articles/organizing-information-with-tables/ +"GitHub Help: Organizing information with tables") and a custom citation syntax. +Check out [Tables Generator](http://www.tablesgenerator.com/markdown_tables) for +creating markdown tables. + +The custom citation guidelines are as follows: + +1. Always use a DOI for the version of record if available. Cite DOIs like + `[@doi:10.15363/thinklab.4]`. ++ If the version of record doesn't have a DOI but does have a PubMed ID, cite + like `[@pmid:26158728]`. ++ If the article is an _arXiv_ preprint, cite like `[@arxiv:1508.06576]`. ++ If the article has none of the above, please file an issue. + +You cite multiple items at once like `[@doi:10.15363/thinklab.4 @pmid:26158728 +@arxiv:1508.06576]`. + +For improved readability, you can (but are not required to) use a tag. For +example, you can add a reference to the tag `Zhou2015_deep_sea` using the +following syntax: `[@tag:Zhou2015_deep_sea]`. If you add references that use +tags, make sure to add those tags and their corresponding citations to +[`references/tags.tsv`](references/tags.tsv). From 5015e16bb4297e563760076851fa5a95e2dbb5d6 Mon Sep 17 00:00:00 2001 From: Anthony Gitter Date: Sun, 13 Nov 2016 15:10:45 -0600 Subject: [PATCH 017/216] URL citation guidelines --- CONTRIBUTING.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0fea540..e723de3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -51,7 +51,8 @@ The custom citation guidelines are as follows: + If the version of record doesn't have a DOI but does have a PubMed ID, cite like `[@pmid:26158728]`. + If the article is an _arXiv_ preprint, cite like `[@arxiv:1508.06576]`. -+ If the article has none of the above, please file an issue. ++ If and only if the article has none of the above, cite with the URL like + `[@url:http://openreview.net/pdf?id=Sk-oDY9ge]`. You cite multiple items at once like `[@doi:10.15363/thinklab.4 @pmid:26158728 @arxiv:1508.06576]`. From 4971905ef79ff13c04cdc321dc2d23dc4d9961ad Mon Sep 17 00:00:00 2001 From: Casey Greene Date: Wed, 21 Dec 2016 09:52:30 -0500 Subject: [PATCH 018/216] 12/21 Status Report --- README.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index f820d9e..1c9866e 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ We have the opportunity to write a headline review for *Journal of the Royal Society Interface* on a topic overlapping the computer and life sciences in the area of systems pharmacology. Our aim is to have the paper ready to submit by -*December 1, 2016*. +*January 15, 2016*. > A Headline Review is one in a short, targeted series of high-level reviews within a particular topic of a burgeoning research area. We encourage authors to @@ -33,6 +33,12 @@ model. Anyone whose contributions meet the ICJME standards of authorship will be included as an author on the manuscript. I can't guarantee that it will be accepted, but I look forward to trying this approach out. +## Status Report on 12/21 + +We are now actively writing the review. Markdown files can be found in the +`sections/` folder. Please claim a section, create a fork, and contribute that +section back via a pull request. + ## Status Report on 10/26 We are now actively outlining the review sections and will begin writing From 5dd62cc84ecce0e6ffea0b19d8202f2922bc61da Mon Sep 17 00:00:00 2001 From: Casey Greene Date: Wed, 21 Dec 2016 09:53:17 -0500 Subject: [PATCH 019/216] 2016 -> 2017 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1c9866e..b203842 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ We have the opportunity to write a headline review for *Journal of the Royal Society Interface* on a topic overlapping the computer and life sciences in the area of systems pharmacology. Our aim is to have the paper ready to submit by -*January 15, 2016*. +*January 15, 2017*. > A Headline Review is one in a short, targeted series of high-level reviews within a particular topic of a burgeoning research area. We encourage authors to From b9e856ca9827a3f0d51c7c85e5e14f6d32b95fc1 Mon Sep 17 00:00:00 2001 From: Casey Greene Date: Wed, 21 Dec 2016 10:47:57 -0500 Subject: [PATCH 020/216] link example PR --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index b203842..4ca60f6 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,9 @@ accepted, but I look forward to trying this approach out. We are now actively writing the review. Markdown files can be found in the `sections/` folder. Please claim a section, create a fork, and contribute that -section back via a pull request. +section back via a pull request. To see what a pull request into the paper +entails, check out [PR #147](https://github.com/greenelab/deep-review/pull/147) +from @evancofer. ## Status Report on 10/26 From 01e059449231aa620d692c72ae67dfbcfb02666c Mon Sep 17 00:00:00 2001 From: Casey Greene Date: Wed, 21 Dec 2016 10:48:44 -0500 Subject: [PATCH 021/216] link evan --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4ca60f6..70da99d 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ We are now actively writing the review. Markdown files can be found in the `sections/` folder. Please claim a section, create a fork, and contribute that section back via a pull request. To see what a pull request into the paper entails, check out [PR #147](https://github.com/greenelab/deep-review/pull/147) -from @evancofer. +from [@evancofer](https://github.com/evancofer). ## Status Report on 10/26 From 873336817a24be59eb1f9a62a3c682b53043fd72 Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Fri, 24 Feb 2017 14:20:37 -0500 Subject: [PATCH 022/216] Add ignores --- .gitignore | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a2d5e12 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +# Python +__pycache__/ +*.pyc + +# Jupyter Notebook +.ipynb_checkpoints From a8ec509a746814ef5fc63853717621b2c1106f31 Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Fri, 24 Feb 2017 15:25:23 -0500 Subject: [PATCH 023/216] Add ability to autogenerate citations --- build.sh | 2 - build/README.md | 14 ++ build/build.sh | 31 +++ build/citations.py | 156 ++++++++++++++ build/environment.yml | 14 ++ build/metadata.py | 101 +++++++++ build/references.ipynb | 454 +++++++++++++++++++++++++++++++++++++++++ 7 files changed, 770 insertions(+), 2 deletions(-) delete mode 100644 build.sh create mode 100644 build/README.md create mode 100644 build/build.sh create mode 100644 build/citations.py create mode 100644 build/environment.yml create mode 100644 build/metadata.py create mode 100644 build/references.ipynb diff --git a/build.sh b/build.sh deleted file mode 100644 index 8e79993..0000000 --- a/build.sh +++ /dev/null @@ -1,2 +0,0 @@ - -pandoc --verbose --from=markdown --output=review.pdf sections/*.md diff --git a/build/README.md b/build/README.md new file mode 100644 index 0000000..ea6430a --- /dev/null +++ b/build/README.md @@ -0,0 +1,14 @@ +# Building the deep-review manuscript + +[`build.sh`](build.sh) builds the repository. +`sh build.sh` should be executed from the root directory of the repository. + +## Environment + +Install the [conda](https://conda.io) environment specified in [`environment.yml`](../environment.yml) by running: + +```sh +conda env create --file environment.yml +``` + +Activate with `source activate deep-review`. diff --git a/build/build.sh b/build/build.sh new file mode 100644 index 0000000..6f91512 --- /dev/null +++ b/build/build.sh @@ -0,0 +1,31 @@ +set -o errexit + +# For local convertsion to PDF without any citations use: +# pandoc --verbose --from=markdown --output=review.pdf sections/*.md + +jupyter nbconvert --inplace \ + --execute --ExecutePreprocessor.timeout=-1 \ + build/references.ipynb + +# pandoc settings +CSL_PATH=https://github.com/citation-style-language/styles/raw/master/plos.csl +BIBLIOGRAPHY_PATH=references/generated/bibliography.json +INPUT_PATH=references/generated/all-sections.md + +# Create HTML outpout +# http://pandoc.org/MANUAL.html +pandoc --verbose \ + --from=markdown --to=html \ + --bibliography=$BIBLIOGRAPHY_PATH \ + --csl=$CSL_PATH \ + --output=review.html \ + $INPUT_PATH + +# Create PDF outpout +pandoc \ + --from=markdown \ + --to=html5 \ + --bibliography=$BIBLIOGRAPHY_PATH \ + --csl=$CSL_PATH \ + --output=review.pdf \ + $INPUT_PATH diff --git a/build/citations.py b/build/citations.py new file mode 100644 index 0000000..f5e3b99 --- /dev/null +++ b/build/citations.py @@ -0,0 +1,156 @@ +import collections +import pathlib +import re + +import pandas +import bibtexparser + +import metadata + + +def validate_reference(ref): + if not ref.startswith('@'): + return f'{ref} does not start with @' + source, tail = ref.lstrip('@').split(':', 1) + if not tail: + return f'{ref} does not specify its source, e.g. doi or pmid' + if source not in {'doi', 'pmid', 'arxiv', 'url', 'tag'}: + return f'{ref} source is not valid' + return None + + +def get_references_from_text(text): + """ + Extract the set of references in a text + """ + refs = set() + for ref_text in re.findall(r'\[(@.+?)\]', text, flags=re.DOTALL): + for ref in ref_text.split(): + if not ref: + continue + refs.add(ref) + return refs + + +def get_text(directory): + """ + Return a dictionary of section texts in the specified directory. + """ + section_dir = pathlib.Path(directory) + paths = sorted(section_dir.iterdir()) + name_to_text = collections.OrderedDict() + for path in paths: + if not re.match(r'[0-9]', path.name): + continue + with path.open('rt') as section_file: + name_to_text[path.stem] = section_file.read() + return '\n\n'.join(name_to_text.values()) + + +def standardize_identifier(source, identifier): + """ + Standardize idenfiers based on their source + """ + if source == 'doi': + identifier = identifier.lower() + return identifier + + +def citation_to_metadata(citation, cache={}): + """ + Return a dictionary with citation metadata + """ + source, identifer = citation.split(':', 1) + identifer = standardize_identifier(source, identifer) + standard_citation = f'{source}:{identifer}' + if standard_citation in cache: + return cache[standard_citation] + + result = { + 'source': source, + 'identifer': identifer, + 'standard_citation': standard_citation + } + + if source == 'doi': + result['citeproc'] = metadata.get_doi_citeproc(identifer) + # result['bibtex'] = metadata.get_doi_bibtex(identifer) + elif source == 'pmid': + result['citeproc'] = metadata.get_pubmed_citeproc(identifer) + elif source == 'arxiv': + result['bibtex'] = metadata.get_arxiv_bibtex(identifer) + elif source == 'url': + result['citeproc'] = metadata.get_url_citeproc(identifer) + else: + msg = f'Unsupported citation source {source} in {citation}' + raise ValueError(msg) + + citation_id = f'ref_{len(cache)}' + result['citation_id'] = citation_id + if 'citeproc' in result: + result['citeproc'] = citeproc_passthrough(result['citeproc'], set_id=citation_id) + if 'bibtex' in result: + result['bibtex'] = bibtex_passthrough(result['bibtex'], set_id=citation_id) + + cache[standard_citation] = result + return result + + +citeproc_type_fixer = { + 'journal-article': 'article-journal', + 'book-chapter': 'chapter', + 'posted-content': 'manuscript', + 'proceedings-article': 'paper-conference', +} + +citeproc_remove_keys = [ + # pandoc-citeproc: Error in $[0].ISSN[0]: failed to parse field ISSN: mempty + 'ISSN', + # pandoc-citeproc: Error in $[2].ISBN[0]: failed to parse field ISBN: mempty + 'ISBN', + # pandoc-citeproc expected Object not array for archive + 'archive', +] + +def citeproc_passthrough(csl_item, set_id=None): + """ + Fix errors in a CSL item and optional change its id. + http://citeproc-js.readthedocs.io/en/latest/csl-json/markup.html + https://github.com/citation-style-language/schema/blob/master/csl-data.json + """ + if set_id is not None: + csl_item['id'] = set_id + + # Correct invalid CSL item types + old_type = csl_item['type'] + csl_item['type'] = citeproc_type_fixer.get(old_type, old_type) + + # Remove problematic objects + for key in citeproc_remove_keys: + csl_item.pop(key, None) + + # pandoc-citeproc error + # failed to parse field issued: Could not read as string: Null + try: + value = csl_item['issued']['date-parts'][0][0] + if value is None: + del csl_item['issued'] + except KeyError: + pass + + return csl_item + + +def bibtex_passthrough(text, set_id=None): + """ + Fix errors in a bibtex record and optional change its ID. + """ + parser = bibtexparser.bparser.BibTexParser() + parser.ignore_nonstandard_types = False + bibdb = bibtexparser.loads(text, parser) + entry, = bibdb.entries + if 'author' in entry: + entry['author'] = ' and '.join(entry['author'].rstrip(';').split('; ')) + if set_id is not None: + entry['ID'] = set_id + return bibtexparser.dumps(bibdb) diff --git a/build/environment.yml b/build/environment.yml new file mode 100644 index 0000000..0aabab9 --- /dev/null +++ b/build/environment.yml @@ -0,0 +1,14 @@ +name: deepreview +channels: + - anaconda + - conda-forge +dependencies: + - bioconda::wkhtmltopdf=0.12.3 + - conda-forge::pandoc=1.19.2 + - jupyter=1.0.0 + - pandas=0.19.2 + - python=3.6.0 + - requests=2.13.0 + - pip: + - arxiv2bib==1.0.7 + - bibtexparser==0.6.2 diff --git a/build/metadata.py b/build/metadata.py new file mode 100644 index 0000000..cd3c1eb --- /dev/null +++ b/build/metadata.py @@ -0,0 +1,101 @@ +import urllib.request + +import requests + + +def get_short_doi(doi, cache={}, verbose=False): + """ + Get the shortDOI for a DOI. Providing a cache dictionary will prevent + multiple API requests for the same DOI. + """ + if doi in cache: + return cache[doi] + quoted_doi = urllib.request.quote(doi) + url = 'http://shortdoi.org/{}?format=json'.format(quoted_doi) + try: + response = requests.get(url).json() + short_doi = response['ShortDOI'] + except Exception as e: + if verbose: + print(doi, 'failed with', e) + return None + cache[doi] = short_doi + return short_doi + + +def get_doi_citeproc(doi): + """ + Use Content Negotioation (http://citation.crosscite.org/docs.html) to + retrieve the citeproc JSON citation for a DOI. + """ + url = 'https://doi.org/' + urllib.request.quote(doi) + header = { + 'Accept': 'application/vnd.citationstyles.csl+json', + } + response = requests.get(url, headers=header) + return response.json() + + +def get_pubmed_citeproc(pubmed_id): + """ + Get the citeproc JSON for a PubMed or PubMed Central identifier + + https://github.com/ncbi/citation-exporter + https://www.ncbi.nlm.nih.gov/pmc/tools/ctxp/ + https://www.ncbi.nlm.nih.gov/pmc/utils/ctxp/samples + """ + params = { + 'ids': pubmed_id, + 'report': 'citeproc' + } + url = 'https://www.ncbi.nlm.nih.gov/pmc/utils/ctxp' + response = requests.get(url, params) + return response.json() + + +# def get_url_citeproc(url): +# """ +# Uses greycite (unreliable) +# https://github.com/cboettig/knitcitations/blob/master/R/greycite.R +# """ +# params = {'uri': url} +# url = 'http://greycite.knowledgeblog.org/' +# response = requests.get(url, params) +# print(response.url) +# return response + + +def get_url_citeproc(url): + """ + Uses greycite (unreliable) + https://github.com/cboettig/knitcitations/blob/master/R/greycite.R + """ + return { + 'URL': url, + 'type': 'webpage' + } + + +def get_doi_bibtex(doi): + """ + Use DOI Content Negotioation (http://crosscite.org/cn/) to retrieve a + string with the bibtex entry. + """ + url = 'https://doi.org/' + urllib.request.quote(doi) + header = { + 'Accept': 'application/x-bibtex', + } + response = requests.get(url, headers=header) + bibtext = response.text + return bibtext + + +def get_arxiv_bibtex(arxiv_id): + """ + Use the arxiv2bib package to get the BibTex for an arXiv preprint. + """ + import arxiv2bib + ref, = arxiv2bib.arxiv2bib([arxiv_id]) + if isinstance(ref, arxiv2bib.ReferenceErrorInfo): + raise KeyError(f'{ref.message}: {arxiv_id}') + return ref.bibtex() diff --git a/build/references.ipynb b/build/references.ipynb new file mode 100644 index 0000000..3a07d3e --- /dev/null +++ b/build/references.ipynb @@ -0,0 +1,454 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "deletable": true, + "editable": true + }, + "source": [ + "# Process citations and retrieve metadata" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": false, + "deletable": true, + "editable": true + }, + "outputs": [], + "source": [ + "import json\n", + "import re\n", + "import os\n", + "import pathlib\n", + "import subprocess\n", + "\n", + "import pandas\n", + "\n", + "from citations import (\n", + " citation_to_metadata,\n", + " citeproc_passthrough,\n", + " get_references_from_text,\n", + " get_text,\n", + " validate_reference,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "collapsed": true, + "deletable": true, + "editable": true + }, + "outputs": [], + "source": [ + "ref_dir = pathlib.Path('../references')\n", + "gen_dir = ref_dir.joinpath('generated')" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "collapsed": false, + "deletable": true, + "editable": true + }, + "outputs": [], + "source": [ + "text = get_text('../sections')\n", + "refs = sorted(get_references_from_text(text))\n", + "bad_refs = list(filter(None, map(validate_reference, refs)))\n", + "if bad_refs:\n", + " print('\\n'.join(bad_refs))\n", + " assert False\n", + "ref_df = pandas.DataFrame({'text': refs})\n", + "tag_df = pandas.read_table(ref_dir.joinpath('tags.tsv'))\n", + "tag_df['text'] = '@tag:' + tag_df.tag\n", + "ref_df = ref_df.merge(tag_df[['text', 'citation']], how='left')\n", + "ref_df.citation.fillna(ref_df.text.str.lstrip('@'), inplace=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "collapsed": false, + "deletable": true, + "editable": true + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
textcitation
0@arxiv:1409.0575arxiv:1409.0575
1@arxiv:1411.2581v1arxiv:1411.2581v1
2@arxiv:1510.02855arxiv:1510.02855
\n", + "
" + ], + "text/plain": [ + " text citation\n", + "0 @arxiv:1409.0575 arxiv:1409.0575\n", + "1 @arxiv:1411.2581v1 arxiv:1411.2581v1\n", + "2 @arxiv:1510.02855 arxiv:1510.02855" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ref_df.head(3)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "collapsed": false, + "deletable": true, + "editable": true + }, + "outputs": [], + "source": [ + "def get_standard_citatation(citation, cache):\n", + " try:\n", + " metadata = citation_to_metadata(citation, cache)\n", + " return metadata['standard_citation'], metadata['citation_id']\n", + " except Exception as e:\n", + " print(citation, e)\n", + " return None, None" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "collapsed": false, + "deletable": true, + "editable": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Using metadata cache: False\n" + ] + } + ], + "source": [ + "cache_path = gen_dir.joinpath('citations.json')\n", + "use_cache = cache_path.exists() and 'REFRESH_METADATA_CACHE' not in os.environ\n", + "print('Using metadata cache:', use_cache)\n", + "if use_cache:\n", + " with gen_dir.joinpath('citations.json').open() as read_file:\n", + " metadata_cache = json.load(read_file)\n", + "else:\n", + " metadata_cache = {}\n", + "\n", + "ref_df['standard_citation'], ref_df['citation_id'] = zip(*ref_df.citation.apply(\n", + " get_standard_citatation, cache=metadata_cache))" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "collapsed": false, + "deletable": true, + "editable": true + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
textcitationstandard_citationcitation_id
0@arxiv:1409.0575arxiv:1409.0575arxiv:1409.0575ref_0
1@arxiv:1411.2581v1arxiv:1411.2581v1arxiv:1411.2581v1ref_1
2@arxiv:1510.02855arxiv:1510.02855arxiv:1510.02855ref_2
\n", + "
" + ], + "text/plain": [ + " text citation standard_citation citation_id\n", + "0 @arxiv:1409.0575 arxiv:1409.0575 arxiv:1409.0575 ref_0\n", + "1 @arxiv:1411.2581v1 arxiv:1411.2581v1 arxiv:1411.2581v1 ref_1\n", + "2 @arxiv:1510.02855 arxiv:1510.02855 arxiv:1510.02855 ref_2" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ref_df.head(3)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "collapsed": false, + "deletable": true, + "editable": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "118 unique citations strings extracted from text\n", + "116 unique citations after standardizations\n" + ] + } + ], + "source": [ + "print(f'''\n", + "{len(ref_df)} unique citations strings extracted from text\n", + "{ref_df.standard_citation.nunique()} unique citations after standardizations\n", + "'''.strip())" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "collapsed": false, + "deletable": true, + "editable": true + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
textcitationstandard_citationcitation_id
26@doi:10.1016/j.neunet.2014.09.003doi:10.1016/j.neunet.2014.09.003doi:10.1016/j.neunet.2014.09.003ref_26
46@doi:10.1101/073239doi:10.1101/073239doi:10.1101/073239ref_46
105@tag:Schmidhuber2014_dnn_overviewdoi:10.1016/j.neunet.2014.09.003doi:10.1016/j.neunet.2014.09.003ref_26
113@tag:Wang2016_protein_contactdoi:10.1101/073239doi:10.1101/073239ref_46
\n", + "
" + ], + "text/plain": [ + " text citation \\\n", + "26 @doi:10.1016/j.neunet.2014.09.003 doi:10.1016/j.neunet.2014.09.003 \n", + "46 @doi:10.1101/073239 doi:10.1101/073239 \n", + "105 @tag:Schmidhuber2014_dnn_overview doi:10.1016/j.neunet.2014.09.003 \n", + "113 @tag:Wang2016_protein_contact doi:10.1101/073239 \n", + "\n", + " standard_citation citation_id \n", + "26 doi:10.1016/j.neunet.2014.09.003 ref_26 \n", + "46 doi:10.1101/073239 ref_46 \n", + "105 doi:10.1016/j.neunet.2014.09.003 ref_26 \n", + "113 doi:10.1101/073239 ref_46 " + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Duplicated citations\n", + "ref_df[ref_df.standard_citation.duplicated(keep=False)]" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "collapsed": false, + "deletable": true, + "editable": true + }, + "outputs": [], + "source": [ + "converted_text = text\n", + "for old, new in zip(ref_df.text, '@' + ref_df.citation_id):\n", + " old = re.escape(old)\n", + " converted_text = re.sub(old + '(?=[\\s\\]])', new, converted_text)\n", + "\n", + "with gen_dir.joinpath('all-sections.md').open('wt') as write_file:\n", + " write_file.write(converted_text)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "collapsed": false, + "deletable": true, + "editable": true + }, + "outputs": [], + "source": [ + "path = gen_dir.joinpath('processed-citations.tsv')\n", + "ref_df.to_csv(path, sep='\\t', index=False)\n", + "\n", + "with cache_path.open('wt') as write_file:\n", + " json.dump(metadata_cache, write_file, indent=2, ensure_ascii=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "collapsed": false, + "deletable": true, + "editable": true + }, + "outputs": [], + "source": [ + "csl_items = list()\n", + "bibtex_stanzas = list()\n", + "for metadata in metadata_cache.values():\n", + " if 'citeproc' in metadata:\n", + " csl_items.append(metadata['citeproc'])\n", + " elif 'bibtex' in metadata:\n", + " bibtex_stanzas.append(metadata['bibtex'])\n", + "\n", + "bib_path = gen_dir.joinpath('bibliography.bib')\n", + "with bib_path.open('wt') as write_file:\n", + " write_file.write('\\n'.join(bibtex_stanzas))\n", + "\n", + "bib_items = subprocess.check_output(['pandoc-citeproc', '--bib2json', bib_path])\n", + "bib_items = json.loads(bib_items)\n", + "csl_items.extend(map(citeproc_passthrough, bib_items))" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "collapsed": false, + "deletable": true, + "editable": true + }, + "outputs": [], + "source": [ + "with gen_dir.joinpath('bibliography.json').open('wt') as write_file:\n", + " json.dump(csl_items, write_file, indent=2, ensure_ascii=False)" + ] + } + ], + "metadata": { + "anaconda-cloud": {}, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.0" + } + }, + "nbformat": 4, + "nbformat_minor": 1 +} From ecef917cff25f0b46b4d0b6380e3260f12534429 Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Fri, 24 Feb 2017 15:32:38 -0500 Subject: [PATCH 024/216] Enable Travis CI --- .travis.yml | 15 +++++++++++++++ README.md | 5 +++++ 2 files changed, 20 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..49a3172 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,15 @@ +language: generic +before_install: + - sudo apt-get update --quiet --quiet + - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh --output-document miniconda.sh + - bash miniconda.sh -b -p $HOME/miniconda + - export PATH="$HOME/miniconda/bin:$PATH" + - hash -r + - conda config --set always_yes yes --set changeps1 no + - conda update --quiet conda + - conda info --all +install: + - conda env create --quiet --file build/environment.yml + - source activate deepreview +script: + - sh build/build.sh diff --git a/README.md b/README.md index 70da99d..0813945 100644 --- a/README.md +++ b/README.md @@ -80,3 +80,8 @@ argue about the answer to the question that we coalesce on with [#88](https://github.com/greenelab/deep-review/issues/88) for each area that the review will cover. +## Continuous Integration + +[![Build Status](https://travis-ci.org/greenelab/deep-review.svg?branch=master)](https://travis-ci.org/greenelab/deep-review) + +[`.travis.yml`](.travis.yml) configures the continuous integration. From abf44742c858074947ff63cc0206bd8c11919a05 Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Fri, 24 Feb 2017 15:51:35 -0500 Subject: [PATCH 025/216] Switch from notebook to script --- build/build.sh | 4 +- build/references.ipynb | 454 ----------------------------------------- build/references.py | 139 +++++++++++++ 3 files changed, 140 insertions(+), 457 deletions(-) delete mode 100644 build/references.ipynb create mode 100644 build/references.py diff --git a/build/build.sh b/build/build.sh index 6f91512..813c5aa 100644 --- a/build/build.sh +++ b/build/build.sh @@ -3,9 +3,7 @@ set -o errexit # For local convertsion to PDF without any citations use: # pandoc --verbose --from=markdown --output=review.pdf sections/*.md -jupyter nbconvert --inplace \ - --execute --ExecutePreprocessor.timeout=-1 \ - build/references.ipynb +(cd build && python references.py) # pandoc settings CSL_PATH=https://github.com/citation-style-language/styles/raw/master/plos.csl diff --git a/build/references.ipynb b/build/references.ipynb deleted file mode 100644 index 3a07d3e..0000000 --- a/build/references.ipynb +++ /dev/null @@ -1,454 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, - "source": [ - "# Process citations and retrieve metadata" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, - "outputs": [], - "source": [ - "import json\n", - "import re\n", - "import os\n", - "import pathlib\n", - "import subprocess\n", - "\n", - "import pandas\n", - "\n", - "from citations import (\n", - " citation_to_metadata,\n", - " citeproc_passthrough,\n", - " get_references_from_text,\n", - " get_text,\n", - " validate_reference,\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "collapsed": true, - "deletable": true, - "editable": true - }, - "outputs": [], - "source": [ - "ref_dir = pathlib.Path('../references')\n", - "gen_dir = ref_dir.joinpath('generated')" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, - "outputs": [], - "source": [ - "text = get_text('../sections')\n", - "refs = sorted(get_references_from_text(text))\n", - "bad_refs = list(filter(None, map(validate_reference, refs)))\n", - "if bad_refs:\n", - " print('\\n'.join(bad_refs))\n", - " assert False\n", - "ref_df = pandas.DataFrame({'text': refs})\n", - "tag_df = pandas.read_table(ref_dir.joinpath('tags.tsv'))\n", - "tag_df['text'] = '@tag:' + tag_df.tag\n", - "ref_df = ref_df.merge(tag_df[['text', 'citation']], how='left')\n", - "ref_df.citation.fillna(ref_df.text.str.lstrip('@'), inplace=True)" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
textcitation
0@arxiv:1409.0575arxiv:1409.0575
1@arxiv:1411.2581v1arxiv:1411.2581v1
2@arxiv:1510.02855arxiv:1510.02855
\n", - "
" - ], - "text/plain": [ - " text citation\n", - "0 @arxiv:1409.0575 arxiv:1409.0575\n", - "1 @arxiv:1411.2581v1 arxiv:1411.2581v1\n", - "2 @arxiv:1510.02855 arxiv:1510.02855" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ref_df.head(3)" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, - "outputs": [], - "source": [ - "def get_standard_citatation(citation, cache):\n", - " try:\n", - " metadata = citation_to_metadata(citation, cache)\n", - " return metadata['standard_citation'], metadata['citation_id']\n", - " except Exception as e:\n", - " print(citation, e)\n", - " return None, None" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Using metadata cache: False\n" - ] - } - ], - "source": [ - "cache_path = gen_dir.joinpath('citations.json')\n", - "use_cache = cache_path.exists() and 'REFRESH_METADATA_CACHE' not in os.environ\n", - "print('Using metadata cache:', use_cache)\n", - "if use_cache:\n", - " with gen_dir.joinpath('citations.json').open() as read_file:\n", - " metadata_cache = json.load(read_file)\n", - "else:\n", - " metadata_cache = {}\n", - "\n", - "ref_df['standard_citation'], ref_df['citation_id'] = zip(*ref_df.citation.apply(\n", - " get_standard_citatation, cache=metadata_cache))" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
textcitationstandard_citationcitation_id
0@arxiv:1409.0575arxiv:1409.0575arxiv:1409.0575ref_0
1@arxiv:1411.2581v1arxiv:1411.2581v1arxiv:1411.2581v1ref_1
2@arxiv:1510.02855arxiv:1510.02855arxiv:1510.02855ref_2
\n", - "
" - ], - "text/plain": [ - " text citation standard_citation citation_id\n", - "0 @arxiv:1409.0575 arxiv:1409.0575 arxiv:1409.0575 ref_0\n", - "1 @arxiv:1411.2581v1 arxiv:1411.2581v1 arxiv:1411.2581v1 ref_1\n", - "2 @arxiv:1510.02855 arxiv:1510.02855 arxiv:1510.02855 ref_2" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ref_df.head(3)" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "118 unique citations strings extracted from text\n", - "116 unique citations after standardizations\n" - ] - } - ], - "source": [ - "print(f'''\n", - "{len(ref_df)} unique citations strings extracted from text\n", - "{ref_df.standard_citation.nunique()} unique citations after standardizations\n", - "'''.strip())" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
textcitationstandard_citationcitation_id
26@doi:10.1016/j.neunet.2014.09.003doi:10.1016/j.neunet.2014.09.003doi:10.1016/j.neunet.2014.09.003ref_26
46@doi:10.1101/073239doi:10.1101/073239doi:10.1101/073239ref_46
105@tag:Schmidhuber2014_dnn_overviewdoi:10.1016/j.neunet.2014.09.003doi:10.1016/j.neunet.2014.09.003ref_26
113@tag:Wang2016_protein_contactdoi:10.1101/073239doi:10.1101/073239ref_46
\n", - "
" - ], - "text/plain": [ - " text citation \\\n", - "26 @doi:10.1016/j.neunet.2014.09.003 doi:10.1016/j.neunet.2014.09.003 \n", - "46 @doi:10.1101/073239 doi:10.1101/073239 \n", - "105 @tag:Schmidhuber2014_dnn_overview doi:10.1016/j.neunet.2014.09.003 \n", - "113 @tag:Wang2016_protein_contact doi:10.1101/073239 \n", - "\n", - " standard_citation citation_id \n", - "26 doi:10.1016/j.neunet.2014.09.003 ref_26 \n", - "46 doi:10.1101/073239 ref_46 \n", - "105 doi:10.1016/j.neunet.2014.09.003 ref_26 \n", - "113 doi:10.1101/073239 ref_46 " - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# Duplicated citations\n", - "ref_df[ref_df.standard_citation.duplicated(keep=False)]" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, - "outputs": [], - "source": [ - "converted_text = text\n", - "for old, new in zip(ref_df.text, '@' + ref_df.citation_id):\n", - " old = re.escape(old)\n", - " converted_text = re.sub(old + '(?=[\\s\\]])', new, converted_text)\n", - "\n", - "with gen_dir.joinpath('all-sections.md').open('wt') as write_file:\n", - " write_file.write(converted_text)" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, - "outputs": [], - "source": [ - "path = gen_dir.joinpath('processed-citations.tsv')\n", - "ref_df.to_csv(path, sep='\\t', index=False)\n", - "\n", - "with cache_path.open('wt') as write_file:\n", - " json.dump(metadata_cache, write_file, indent=2, ensure_ascii=False)" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, - "outputs": [], - "source": [ - "csl_items = list()\n", - "bibtex_stanzas = list()\n", - "for metadata in metadata_cache.values():\n", - " if 'citeproc' in metadata:\n", - " csl_items.append(metadata['citeproc'])\n", - " elif 'bibtex' in metadata:\n", - " bibtex_stanzas.append(metadata['bibtex'])\n", - "\n", - "bib_path = gen_dir.joinpath('bibliography.bib')\n", - "with bib_path.open('wt') as write_file:\n", - " write_file.write('\\n'.join(bibtex_stanzas))\n", - "\n", - "bib_items = subprocess.check_output(['pandoc-citeproc', '--bib2json', bib_path])\n", - "bib_items = json.loads(bib_items)\n", - "csl_items.extend(map(citeproc_passthrough, bib_items))" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, - "outputs": [], - "source": [ - "with gen_dir.joinpath('bibliography.json').open('wt') as write_file:\n", - " json.dump(csl_items, write_file, indent=2, ensure_ascii=False)" - ] - } - ], - "metadata": { - "anaconda-cloud": {}, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.0" - } - }, - "nbformat": 4, - "nbformat_minor": 1 -} diff --git a/build/references.py b/build/references.py new file mode 100644 index 0000000..ab1fd3c --- /dev/null +++ b/build/references.py @@ -0,0 +1,139 @@ + +# coding: utf-8 + +# # Process citations and retrieve metadata + +# In[1]: + +import json +import re +import os +import pathlib +import subprocess + +import pandas + +from citations import ( + citation_to_metadata, + citeproc_passthrough, + get_references_from_text, + get_text, + validate_reference, +) + + +# In[2]: + +ref_dir = pathlib.Path('../references') +gen_dir = ref_dir.joinpath('generated') + + +# In[3]: + +text = get_text('../sections') +refs = sorted(get_references_from_text(text)) +bad_refs = list(filter(None, map(validate_reference, refs))) +if bad_refs: + print('\n'.join(bad_refs)) + assert False +ref_df = pandas.DataFrame({'text': refs}) +tag_df = pandas.read_table(ref_dir.joinpath('tags.tsv')) +tag_df['text'] = '@tag:' + tag_df.tag +ref_df = ref_df.merge(tag_df[['text', 'citation']], how='left') +ref_df.citation.fillna(ref_df.text.str.lstrip('@'), inplace=True) + + +# In[4]: + +ref_df.head(3) + + +# In[5]: + +def get_standard_citatation(citation, cache): + try: + metadata = citation_to_metadata(citation, cache) + return metadata['standard_citation'], metadata['citation_id'] + except Exception as e: + print(citation, e) + return None, None + + +# In[6]: + +cache_path = gen_dir.joinpath('citations.json') +use_cache = cache_path.exists() and 'REFRESH_METADATA_CACHE' not in os.environ +print('Using metadata cache:', use_cache) +if use_cache: + with gen_dir.joinpath('citations.json').open() as read_file: + metadata_cache = json.load(read_file) +else: + metadata_cache = {} + +ref_df['standard_citation'], ref_df['citation_id'] = zip(*ref_df.citation.apply( + get_standard_citatation, cache=metadata_cache)) + + +# In[7]: + +ref_df.head(3) + + +# In[8]: + +print(f''' +{len(ref_df)} unique citations strings extracted from text +{ref_df.standard_citation.nunique()} unique citations after standardizations +'''.strip()) + + +# In[9]: + +# Duplicated citations +dup_df = ref_df[ref_df.standard_citation.duplicated(keep=False)] +print(dup_df) + +# In[10]: + +converted_text = text +for old, new in zip(ref_df.text, '@' + ref_df.citation_id): + old = re.escape(old) + converted_text = re.sub(old + '(?=[\s\]])', new, converted_text) + +with gen_dir.joinpath('all-sections.md').open('wt') as write_file: + write_file.write(converted_text) + + +# In[11]: + +path = gen_dir.joinpath('processed-citations.tsv') +ref_df.to_csv(path, sep='\t', index=False) + +with cache_path.open('wt') as write_file: + json.dump(metadata_cache, write_file, indent=2, ensure_ascii=False) + + +# In[12]: + +csl_items = list() +bibtex_stanzas = list() +for metadata in metadata_cache.values(): + if 'citeproc' in metadata: + csl_items.append(metadata['citeproc']) + elif 'bibtex' in metadata: + bibtex_stanzas.append(metadata['bibtex']) + +bib_path = gen_dir.joinpath('bibliography.bib') +with bib_path.open('wt') as write_file: + write_file.write('\n'.join(bibtex_stanzas)) + +bib_items = subprocess.check_output(['pandoc-citeproc', '--bib2json', bib_path]) +bib_items = json.loads(bib_items) +csl_items.extend(map(citeproc_passthrough, bib_items)) + + +# In[13]: + +with gen_dir.joinpath('bibliography.json').open('wt') as write_file: + json.dump(csl_items, write_file, indent=2, ensure_ascii=False) + From 78c98b2796b62bea0d89ad3135ceb215954af8c9 Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Fri, 24 Feb 2017 16:02:08 -0500 Subject: [PATCH 026/216] Switch to J R Soc Interface CSL --- build/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/build.sh b/build/build.sh index 813c5aa..da860e4 100644 --- a/build/build.sh +++ b/build/build.sh @@ -6,7 +6,7 @@ set -o errexit (cd build && python references.py) # pandoc settings -CSL_PATH=https://github.com/citation-style-language/styles/raw/master/plos.csl +CSL_PATH=https://github.com/citation-style-language/styles/raw/master/dependent/journal-of-the-royal-society-interface.csl BIBLIOGRAPHY_PATH=references/generated/bibliography.json INPUT_PATH=references/generated/all-sections.md From ce0e9fd0de0b8e2106c9bf424ec413d448fb9d25 Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Fri, 24 Feb 2017 16:38:47 -0500 Subject: [PATCH 027/216] Ignore all output files --- .gitignore | 4 ++++ build/build.sh | 8 ++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index a2d5e12..8e13d34 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,7 @@ +# Output files +output +references/generated + # Python __pycache__/ *.pyc diff --git a/build/build.sh b/build/build.sh index da860e4..cda3d1c 100644 --- a/build/build.sh +++ b/build/build.sh @@ -3,6 +3,7 @@ set -o errexit # For local convertsion to PDF without any citations use: # pandoc --verbose --from=markdown --output=review.pdf sections/*.md +# Generate reference information (cd build && python references.py) # pandoc settings @@ -10,13 +11,16 @@ CSL_PATH=https://github.com/citation-style-language/styles/raw/master/dependent/ BIBLIOGRAPHY_PATH=references/generated/bibliography.json INPUT_PATH=references/generated/all-sections.md +# Make output directory +mkdir -p output + # Create HTML outpout # http://pandoc.org/MANUAL.html pandoc --verbose \ --from=markdown --to=html \ --bibliography=$BIBLIOGRAPHY_PATH \ --csl=$CSL_PATH \ - --output=review.html \ + --output=output/index.html \ $INPUT_PATH # Create PDF outpout @@ -25,5 +29,5 @@ pandoc \ --to=html5 \ --bibliography=$BIBLIOGRAPHY_PATH \ --csl=$CSL_PATH \ - --output=review.pdf \ + --output=output/deep-review.pdf \ $INPUT_PATH From 39ab303aaf86bf21cb509cffea980ff1ef37c2fd Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Fri, 24 Feb 2017 16:46:34 -0500 Subject: [PATCH 028/216] Add ghp-import --- build/environment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/build/environment.yml b/build/environment.yml index 0aabab9..1613fa6 100644 --- a/build/environment.yml +++ b/build/environment.yml @@ -12,3 +12,4 @@ dependencies: - pip: - arxiv2bib==1.0.7 - bibtexparser==0.6.2 + - ghp-import==0.5.4 From f6ff07ec4a6a3e659a58a1b5163b7b284f68d5cb Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Fri, 24 Feb 2017 16:54:48 -0500 Subject: [PATCH 029/216] Make generated directory --- build/references.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/references.py b/build/references.py index ab1fd3c..2d20e64 100644 --- a/build/references.py +++ b/build/references.py @@ -26,7 +26,7 @@ ref_dir = pathlib.Path('../references') gen_dir = ref_dir.joinpath('generated') - +gen_dir.mkdir(exist_ok=True) # In[3]: From 06c672d85afaa820828eb53de3eaa4ba52d26535 Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Fri, 24 Feb 2017 17:09:37 -0500 Subject: [PATCH 030/216] Improve status reporting --- build/build.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build/build.sh b/build/build.sh index cda3d1c..858d751 100644 --- a/build/build.sh +++ b/build/build.sh @@ -1,9 +1,7 @@ set -o errexit -# For local convertsion to PDF without any citations use: -# pandoc --verbose --from=markdown --output=review.pdf sections/*.md - # Generate reference information +echo "Retrieving and processing reference metadata" (cd build && python references.py) # pandoc settings @@ -16,6 +14,7 @@ mkdir -p output # Create HTML outpout # http://pandoc.org/MANUAL.html +echo "Exporting HTML manuscript" pandoc --verbose \ --from=markdown --to=html \ --bibliography=$BIBLIOGRAPHY_PATH \ @@ -24,6 +23,7 @@ pandoc --verbose \ $INPUT_PATH # Create PDF outpout +echo "Exporting PDF manuscript" pandoc \ --from=markdown \ --to=html5 \ From feef8e2e0e098b654c94c2305671c88660266911 Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Sat, 25 Feb 2017 15:34:15 -0500 Subject: [PATCH 031/216] Remove jupyter from environment --- build/environment.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/build/environment.yml b/build/environment.yml index 1613fa6..8c90e38 100644 --- a/build/environment.yml +++ b/build/environment.yml @@ -5,7 +5,6 @@ channels: dependencies: - bioconda::wkhtmltopdf=0.12.3 - conda-forge::pandoc=1.19.2 - - jupyter=1.0.0 - pandas=0.19.2 - python=3.6.0 - requests=2.13.0 From ce8fd2b9bf020868648e39d3263d7d3c26548e03 Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Sat, 25 Feb 2017 15:44:10 -0500 Subject: [PATCH 032/216] Switch CC BY license to markdown License text from https://github.com/idleberg/Creative-Commons-Markdown/blob/95edd678ba1996e10fa576d23b35900dbb310692/4.0/by.markdown. License title collapsed to a single h1 line. --- LICENSE | 395 ----------------------------------------------------- LICENSE.md | 159 +++++++++++++++++++++ 2 files changed, 159 insertions(+), 395 deletions(-) delete mode 100644 LICENSE create mode 100644 LICENSE.md diff --git a/LICENSE b/LICENSE deleted file mode 100644 index 2f244ac..0000000 --- a/LICENSE +++ /dev/null @@ -1,395 +0,0 @@ -Attribution 4.0 International - -======================================================================= - -Creative Commons Corporation ("Creative Commons") is not a law firm and -does not provide legal services or legal advice. Distribution of -Creative Commons public licenses does not create a lawyer-client or -other relationship. Creative Commons makes its licenses and related -information available on an "as-is" basis. Creative Commons gives no -warranties regarding its licenses, any material licensed under their -terms and conditions, or any related information. Creative Commons -disclaims all liability for damages resulting from their use to the -fullest extent possible. - -Using Creative Commons Public Licenses - -Creative Commons public licenses provide a standard set of terms and -conditions that creators and other rights holders may use to share -original works of authorship and other material subject to copyright -and certain other rights specified in the public license below. The -following considerations are for informational purposes only, are not -exhaustive, and do not form part of our licenses. - - Considerations for licensors: Our public licenses are - intended for use by those authorized to give the public - permission to use material in ways otherwise restricted by - copyright and certain other rights. Our licenses are - irrevocable. Licensors should read and understand the terms - and conditions of the license they choose before applying it. - Licensors should also secure all rights necessary before - applying our licenses so that the public can reuse the - material as expected. Licensors should clearly mark any - material not subject to the license. This includes other CC- - licensed material, or material used under an exception or - limitation to copyright. More considerations for licensors: - wiki.creativecommons.org/Considerations_for_licensors - - Considerations for the public: By using one of our public - licenses, a licensor grants the public permission to use the - licensed material under specified terms and conditions. If - the licensor's permission is not necessary for any reason--for - example, because of any applicable exception or limitation to - copyright--then that use is not regulated by the license. Our - licenses grant only permissions under copyright and certain - other rights that a licensor has authority to grant. Use of - the licensed material may still be restricted for other - reasons, including because others have copyright or other - rights in the material. A licensor may make special requests, - such as asking that all changes be marked or described. - Although not required by our licenses, you are encouraged to - respect those requests where reasonable. More_considerations - for the public: - wiki.creativecommons.org/Considerations_for_licensees - -======================================================================= - -Creative Commons Attribution 4.0 International Public License - -By exercising the Licensed Rights (defined below), You accept and agree -to be bound by the terms and conditions of this Creative Commons -Attribution 4.0 International Public License ("Public License"). To the -extent this Public License may be interpreted as a contract, You are -granted the Licensed Rights in consideration of Your acceptance of -these terms and conditions, and the Licensor grants You such rights in -consideration of benefits the Licensor receives from making the -Licensed Material available under these terms and conditions. - - -Section 1 -- Definitions. - - a. Adapted Material means material subject to Copyright and Similar - Rights that is derived from or based upon the Licensed Material - and in which the Licensed Material is translated, altered, - arranged, transformed, or otherwise modified in a manner requiring - permission under the Copyright and Similar Rights held by the - Licensor. For purposes of this Public License, where the Licensed - Material is a musical work, performance, or sound recording, - Adapted Material is always produced where the Licensed Material is - synched in timed relation with a moving image. - - b. Adapter's License means the license You apply to Your Copyright - and Similar Rights in Your contributions to Adapted Material in - accordance with the terms and conditions of this Public License. - - c. Copyright and Similar Rights means copyright and/or similar rights - closely related to copyright including, without limitation, - performance, broadcast, sound recording, and Sui Generis Database - Rights, without regard to how the rights are labeled or - categorized. For purposes of this Public License, the rights - specified in Section 2(b)(1)-(2) are not Copyright and Similar - Rights. - - d. Effective Technological Measures means those measures that, in the - absence of proper authority, may not be circumvented under laws - fulfilling obligations under Article 11 of the WIPO Copyright - Treaty adopted on December 20, 1996, and/or similar international - agreements. - - e. Exceptions and Limitations means fair use, fair dealing, and/or - any other exception or limitation to Copyright and Similar Rights - that applies to Your use of the Licensed Material. - - f. Licensed Material means the artistic or literary work, database, - or other material to which the Licensor applied this Public - License. - - g. Licensed Rights means the rights granted to You subject to the - terms and conditions of this Public License, which are limited to - all Copyright and Similar Rights that apply to Your use of the - Licensed Material and that the Licensor has authority to license. - - h. Licensor means the individual(s) or entity(ies) granting rights - under this Public License. - - i. Share means to provide material to the public by any means or - process that requires permission under the Licensed Rights, such - as reproduction, public display, public performance, distribution, - dissemination, communication, or importation, and to make material - available to the public including in ways that members of the - public may access the material from a place and at a time - individually chosen by them. - - j. Sui Generis Database Rights means rights other than copyright - resulting from Directive 96/9/EC of the European Parliament and of - the Council of 11 March 1996 on the legal protection of databases, - as amended and/or succeeded, as well as other essentially - equivalent rights anywhere in the world. - - k. You means the individual or entity exercising the Licensed Rights - under this Public License. Your has a corresponding meaning. - - -Section 2 -- Scope. - - a. License grant. - - 1. Subject to the terms and conditions of this Public License, - the Licensor hereby grants You a worldwide, royalty-free, - non-sublicensable, non-exclusive, irrevocable license to - exercise the Licensed Rights in the Licensed Material to: - - a. reproduce and Share the Licensed Material, in whole or - in part; and - - b. produce, reproduce, and Share Adapted Material. - - 2. Exceptions and Limitations. For the avoidance of doubt, where - Exceptions and Limitations apply to Your use, this Public - License does not apply, and You do not need to comply with - its terms and conditions. - - 3. Term. The term of this Public License is specified in Section - 6(a). - - 4. Media and formats; technical modifications allowed. The - Licensor authorizes You to exercise the Licensed Rights in - all media and formats whether now known or hereafter created, - and to make technical modifications necessary to do so. The - Licensor waives and/or agrees not to assert any right or - authority to forbid You from making technical modifications - necessary to exercise the Licensed Rights, including - technical modifications necessary to circumvent Effective - Technological Measures. For purposes of this Public License, - simply making modifications authorized by this Section 2(a) - (4) never produces Adapted Material. - - 5. Downstream recipients. - - a. Offer from the Licensor -- Licensed Material. Every - recipient of the Licensed Material automatically - receives an offer from the Licensor to exercise the - Licensed Rights under the terms and conditions of this - Public License. - - b. No downstream restrictions. You may not offer or impose - any additional or different terms or conditions on, or - apply any Effective Technological Measures to, the - Licensed Material if doing so restricts exercise of the - Licensed Rights by any recipient of the Licensed - Material. - - 6. No endorsement. Nothing in this Public License constitutes or - may be construed as permission to assert or imply that You - are, or that Your use of the Licensed Material is, connected - with, or sponsored, endorsed, or granted official status by, - the Licensor or others designated to receive attribution as - provided in Section 3(a)(1)(A)(i). - - b. Other rights. - - 1. Moral rights, such as the right of integrity, are not - licensed under this Public License, nor are publicity, - privacy, and/or other similar personality rights; however, to - the extent possible, the Licensor waives and/or agrees not to - assert any such rights held by the Licensor to the limited - extent necessary to allow You to exercise the Licensed - Rights, but not otherwise. - - 2. Patent and trademark rights are not licensed under this - Public License. - - 3. To the extent possible, the Licensor waives any right to - collect royalties from You for the exercise of the Licensed - Rights, whether directly or through a collecting society - under any voluntary or waivable statutory or compulsory - licensing scheme. In all other cases the Licensor expressly - reserves any right to collect such royalties. - - -Section 3 -- License Conditions. - -Your exercise of the Licensed Rights is expressly made subject to the -following conditions. - - a. Attribution. - - 1. If You Share the Licensed Material (including in modified - form), You must: - - a. retain the following if it is supplied by the Licensor - with the Licensed Material: - - i. identification of the creator(s) of the Licensed - Material and any others designated to receive - attribution, in any reasonable manner requested by - the Licensor (including by pseudonym if - designated); - - ii. a copyright notice; - - iii. a notice that refers to this Public License; - - iv. a notice that refers to the disclaimer of - warranties; - - v. a URI or hyperlink to the Licensed Material to the - extent reasonably practicable; - - b. indicate if You modified the Licensed Material and - retain an indication of any previous modifications; and - - c. indicate the Licensed Material is licensed under this - Public License, and include the text of, or the URI or - hyperlink to, this Public License. - - 2. You may satisfy the conditions in Section 3(a)(1) in any - reasonable manner based on the medium, means, and context in - which You Share the Licensed Material. For example, it may be - reasonable to satisfy the conditions by providing a URI or - hyperlink to a resource that includes the required - information. - - 3. If requested by the Licensor, You must remove any of the - information required by Section 3(a)(1)(A) to the extent - reasonably practicable. - - 4. If You Share Adapted Material You produce, the Adapter's - License You apply must not prevent recipients of the Adapted - Material from complying with this Public License. - - -Section 4 -- Sui Generis Database Rights. - -Where the Licensed Rights include Sui Generis Database Rights that -apply to Your use of the Licensed Material: - - a. for the avoidance of doubt, Section 2(a)(1) grants You the right - to extract, reuse, reproduce, and Share all or a substantial - portion of the contents of the database; - - b. if You include all or a substantial portion of the database - contents in a database in which You have Sui Generis Database - Rights, then the database in which You have Sui Generis Database - Rights (but not its individual contents) is Adapted Material; and - - c. You must comply with the conditions in Section 3(a) if You Share - all or a substantial portion of the contents of the database. - -For the avoidance of doubt, this Section 4 supplements and does not -replace Your obligations under this Public License where the Licensed -Rights include other Copyright and Similar Rights. - - -Section 5 -- Disclaimer of Warranties and Limitation of Liability. - - a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE - EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS - AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF - ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS, - IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION, - WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR - PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS, - ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT - KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT - ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU. - - b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE - TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION, - NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT, - INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES, - COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR - USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN - ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR - DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR - IN PART, THIS LIMITATION MAY NOT APPLY TO YOU. - - c. The disclaimer of warranties and limitation of liability provided - above shall be interpreted in a manner that, to the extent - possible, most closely approximates an absolute disclaimer and - waiver of all liability. - - -Section 6 -- Term and Termination. - - a. This Public License applies for the term of the Copyright and - Similar Rights licensed here. However, if You fail to comply with - this Public License, then Your rights under this Public License - terminate automatically. - - b. Where Your right to use the Licensed Material has terminated under - Section 6(a), it reinstates: - - 1. automatically as of the date the violation is cured, provided - it is cured within 30 days of Your discovery of the - violation; or - - 2. upon express reinstatement by the Licensor. - - For the avoidance of doubt, this Section 6(b) does not affect any - right the Licensor may have to seek remedies for Your violations - of this Public License. - - c. For the avoidance of doubt, the Licensor may also offer the - Licensed Material under separate terms or conditions or stop - distributing the Licensed Material at any time; however, doing so - will not terminate this Public License. - - d. Sections 1, 5, 6, 7, and 8 survive termination of this Public - License. - - -Section 7 -- Other Terms and Conditions. - - a. The Licensor shall not be bound by any additional or different - terms or conditions communicated by You unless expressly agreed. - - b. Any arrangements, understandings, or agreements regarding the - Licensed Material not stated herein are separate from and - independent of the terms and conditions of this Public License. - - -Section 8 -- Interpretation. - - a. For the avoidance of doubt, this Public License does not, and - shall not be interpreted to, reduce, limit, restrict, or impose - conditions on any use of the Licensed Material that could lawfully - be made without permission under this Public License. - - b. To the extent possible, if any provision of this Public License is - deemed unenforceable, it shall be automatically reformed to the - minimum extent necessary to make it enforceable. If the provision - cannot be reformed, it shall be severed from this Public License - without affecting the enforceability of the remaining terms and - conditions. - - c. No term or condition of this Public License will be waived and no - failure to comply consented to unless expressly agreed to by the - Licensor. - - d. Nothing in this Public License constitutes or may be interpreted - as a limitation upon, or waiver of, any privileges and immunities - that apply to the Licensor or You, including from the legal - processes of any jurisdiction or authority. - - -======================================================================= - -Creative Commons is not a party to its public -licenses. Notwithstanding, Creative Commons may elect to apply one of -its public licenses to material it publishes and in those instances -will be considered the “Licensor.” The text of the Creative Commons -public licenses is dedicated to the public domain under the CC0 Public -Domain Dedication. Except for the limited purpose of indicating that -material is shared under a Creative Commons public license or as -otherwise permitted by the Creative Commons policies published at -creativecommons.org/policies, Creative Commons does not authorize the -use of the trademark "Creative Commons" or any other trademark or logo -of Creative Commons without its prior written consent including, -without limitation, in connection with any unauthorized modifications -to any of its public licenses or any other arrangements, -understandings, or agreements concerning use of licensed material. For -the avoidance of doubt, this paragraph does not form part of the -public licenses. - -Creative Commons may be contacted at creativecommons.org. diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 0000000..75fb1f2 --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,159 @@ +# Creative Commons Attribution 4.0 International + +Creative Commons Corporation (“Creative Commons”) is not a law firm and does not provide legal services or legal advice. Distribution of Creative Commons public licenses does not create a lawyer-client or other relationship. Creative Commons makes its licenses and related information available on an “as-is” basis. Creative Commons gives no warranties regarding its licenses, any material licensed under their terms and conditions, or any related information. Creative Commons disclaims all liability for damages resulting from their use to the fullest extent possible. + +### Using Creative Commons Public Licenses + +Creative Commons public licenses provide a standard set of terms and conditions that creators and other rights holders may use to share original works of authorship and other material subject to copyright and certain other rights specified in the public license below. The following considerations are for informational purposes only, are not exhaustive, and do not form part of our licenses. + +* __Considerations for licensors:__ Our public licenses are intended for use by those authorized to give the public permission to use material in ways otherwise restricted by copyright and certain other rights. Our licenses are irrevocable. Licensors should read and understand the terms and conditions of the license they choose before applying it. Licensors should also secure all rights necessary before applying our licenses so that the public can reuse the material as expected. Licensors should clearly mark any material not subject to the license. This includes other CC-licensed material, or material used under an exception or limitation to copyright. [More considerations for licensors](http://wiki.creativecommons.org/Considerations_for_licensors_and_licensees#Considerations_for_licensors). + +* __Considerations for the public:__ By using one of our public licenses, a licensor grants the public permission to use the licensed material under specified terms and conditions. If the licensor’s permission is not necessary for any reason–for example, because of any applicable exception or limitation to copyright–then that use is not regulated by the license. Our licenses grant only permissions under copyright and certain other rights that a licensor has authority to grant. Use of the licensed material may still be restricted for other reasons, including because others have copyright or other rights in the material. A licensor may make special requests, such as asking that all changes be marked or described. Although not required by our licenses, you are encouraged to respect those requests where reasonable. [More considerations for the public](http://wiki.creativecommons.org/Considerations_for_licensors_and_licensees#Considerations_for_licensees). + +## Creative Commons Attribution 4.0 International Public License + +By exercising the Licensed Rights (defined below), You accept and agree to be bound by the terms and conditions of this Creative Commons Attribution 4.0 International Public License ("Public License"). To the extent this Public License may be interpreted as a contract, You are granted the Licensed Rights in consideration of Your acceptance of these terms and conditions, and the Licensor grants You such rights in consideration of benefits the Licensor receives from making the Licensed Material available under these terms and conditions. + +### Section 1 – Definitions. + +a. __Adapted Material__ means material subject to Copyright and Similar Rights that is derived from or based upon the Licensed Material and in which the Licensed Material is translated, altered, arranged, transformed, or otherwise modified in a manner requiring permission under the Copyright and Similar Rights held by the Licensor. For purposes of this Public License, where the Licensed Material is a musical work, performance, or sound recording, Adapted Material is always produced where the Licensed Material is synched in timed relation with a moving image. + +b. __Adapter's License__ means the license You apply to Your Copyright and Similar Rights in Your contributions to Adapted Material in accordance with the terms and conditions of this Public License. + +c. __Copyright and Similar Rights__ means copyright and/or similar rights closely related to copyright including, without limitation, performance, broadcast, sound recording, and Sui Generis Database Rights, without regard to how the rights are labeled or categorized. For purposes of this Public License, the rights specified in Section 2(b)(1)-(2) are not Copyright and Similar Rights. + +d. __Effective Technological Measures__ means those measures that, in the absence of proper authority, may not be circumvented under laws fulfilling obligations under Article 11 of the WIPO Copyright Treaty adopted on December 20, 1996, and/or similar international agreements. + +e. __Exceptions and Limitations__ means fair use, fair dealing, and/or any other exception or limitation to Copyright and Similar Rights that applies to Your use of the Licensed Material. + +f. __Licensed Material__ means the artistic or literary work, database, or other material to which the Licensor applied this Public License. + +g. __Licensed Rights__ means the rights granted to You subject to the terms and conditions of this Public License, which are limited to all Copyright and Similar Rights that apply to Your use of the Licensed Material and that the Licensor has authority to license. + +h. __Licensor__ means the individual(s) or entity(ies) granting rights under this Public License. + +i. __Share__ means to provide material to the public by any means or process that requires permission under the Licensed Rights, such as reproduction, public display, public performance, distribution, dissemination, communication, or importation, and to make material available to the public including in ways that members of the public may access the material from a place and at a time individually chosen by them. + +j. __Sui Generis Database Rights__ means rights other than copyright resulting from Directive 96/9/EC of the European Parliament and of the Council of 11 March 1996 on the legal protection of databases, as amended and/or succeeded, as well as other essentially equivalent rights anywhere in the world. + +k. __You__ means the individual or entity exercising the Licensed Rights under this Public License. Your has a corresponding meaning. + +### Section 2 – Scope. + +a. ___License grant.___ + + 1. Subject to the terms and conditions of this Public License, the Licensor hereby grants You a worldwide, royalty-free, non-sublicensable, non-exclusive, irrevocable license to exercise the Licensed Rights in the Licensed Material to: + + A. reproduce and Share the Licensed Material, in whole or in part; and + + B. produce, reproduce, and Share Adapted Material. + + 2. __Exceptions and Limitations.__ For the avoidance of doubt, where Exceptions and Limitations apply to Your use, this Public License does not apply, and You do not need to comply with its terms and conditions. + + 3. __Term.__ The term of this Public License is specified in Section 6(a). + + 4. __Media and formats; technical modifications allowed.__ The Licensor authorizes You to exercise the Licensed Rights in all media and formats whether now known or hereafter created, and to make technical modifications necessary to do so. The Licensor waives and/or agrees not to assert any right or authority to forbid You from making technical modifications necessary to exercise the Licensed Rights, including technical modifications necessary to circumvent Effective Technological Measures. For purposes of this Public License, simply making modifications authorized by this Section 2(a)(4) never produces Adapted Material. + + 5. __Downstream recipients.__ + + A. __Offer from the Licensor – Licensed Material.__ Every recipient of the Licensed Material automatically receives an offer from the Licensor to exercise the Licensed Rights under the terms and conditions of this Public License. + + B. __No downstream restrictions.__ You may not offer or impose any additional or different terms or conditions on, or apply any Effective Technological Measures to, the Licensed Material if doing so restricts exercise of the Licensed Rights by any recipient of the Licensed Material. + + 6. __No endorsement.__ Nothing in this Public License constitutes or may be construed as permission to assert or imply that You are, or that Your use of the Licensed Material is, connected with, or sponsored, endorsed, or granted official status by, the Licensor or others designated to receive attribution as provided in Section 3(a)(1)(A)(i). + +b. ___Other rights.___ + + 1. Moral rights, such as the right of integrity, are not licensed under this Public License, nor are publicity, privacy, and/or other similar personality rights; however, to the extent possible, the Licensor waives and/or agrees not to assert any such rights held by the Licensor to the limited extent necessary to allow You to exercise the Licensed Rights, but not otherwise. + + 2. Patent and trademark rights are not licensed under this Public License. + + 3. To the extent possible, the Licensor waives any right to collect royalties from You for the exercise of the Licensed Rights, whether directly or through a collecting society under any voluntary or waivable statutory or compulsory licensing scheme. In all other cases the Licensor expressly reserves any right to collect such royalties. + +### Section 3 – License Conditions. + +Your exercise of the Licensed Rights is expressly made subject to the following conditions. + +a. ___Attribution.___ + + 1. If You Share the Licensed Material (including in modified form), You must: + + A. retain the following if it is supplied by the Licensor with the Licensed Material: + + i. identification of the creator(s) of the Licensed Material and any others designated to receive attribution, in any reasonable manner requested by the Licensor (including by pseudonym if designated); + + ii. a copyright notice; + + iii. a notice that refers to this Public License; + + iv. a notice that refers to the disclaimer of warranties; + + v. a URI or hyperlink to the Licensed Material to the extent reasonably practicable; + + B. indicate if You modified the Licensed Material and retain an indication of any previous modifications; and + + C. indicate the Licensed Material is licensed under this Public License, and include the text of, or the URI or hyperlink to, this Public License. + + 2. You may satisfy the conditions in Section 3(a)(1) in any reasonable manner based on the medium, means, and context in which You Share the Licensed Material. For example, it may be reasonable to satisfy the conditions by providing a URI or hyperlink to a resource that includes the required information. + + 3. If requested by the Licensor, You must remove any of the information required by Section 3(a)(1)(A) to the extent reasonably practicable. + + 4. If You Share Adapted Material You produce, the Adapter's License You apply must not prevent recipients of the Adapted Material from complying with this Public License. + +### Section 4 – Sui Generis Database Rights. + +Where the Licensed Rights include Sui Generis Database Rights that apply to Your use of the Licensed Material: + +a. for the avoidance of doubt, Section 2(a)(1) grants You the right to extract, reuse, reproduce, and Share all or a substantial portion of the contents of the database; + +b. if You include all or a substantial portion of the database contents in a database in which You have Sui Generis Database Rights, then the database in which You have Sui Generis Database Rights (but not its individual contents) is Adapted Material; and + +c. You must comply with the conditions in Section 3(a) if You Share all or a substantial portion of the contents of the database. + +For the avoidance of doubt, this Section 4 supplements and does not replace Your obligations under this Public License where the Licensed Rights include other Copyright and Similar Rights. + +### Section 5 – Disclaimer of Warranties and Limitation of Liability. + +a. __Unless otherwise separately undertaken by the Licensor, to the extent possible, the Licensor offers the Licensed Material as-is and as-available, and makes no representations or warranties of any kind concerning the Licensed Material, whether express, implied, statutory, or other. This includes, without limitation, warranties of title, merchantability, fitness for a particular purpose, non-infringement, absence of latent or other defects, accuracy, or the presence or absence of errors, whether or not known or discoverable. Where disclaimers of warranties are not allowed in full or in part, this disclaimer may not apply to You.__ + +b. __To the extent possible, in no event will the Licensor be liable to You on any legal theory (including, without limitation, negligence) or otherwise for any direct, special, indirect, incidental, consequential, punitive, exemplary, or other losses, costs, expenses, or damages arising out of this Public License or use of the Licensed Material, even if the Licensor has been advised of the possibility of such losses, costs, expenses, or damages. Where a limitation of liability is not allowed in full or in part, this limitation may not apply to You.__ + +c. The disclaimer of warranties and limitation of liability provided above shall be interpreted in a manner that, to the extent possible, most closely approximates an absolute disclaimer and waiver of all liability. + +### Section 6 – Term and Termination. + +a. This Public License applies for the term of the Copyright and Similar Rights licensed here. However, if You fail to comply with this Public License, then Your rights under this Public License terminate automatically. + +b. Where Your right to use the Licensed Material has terminated under Section 6(a), it reinstates: + + 1. automatically as of the date the violation is cured, provided it is cured within 30 days of Your discovery of the violation; or + + 2. upon express reinstatement by the Licensor. + + For the avoidance of doubt, this Section 6(b) does not affect any right the Licensor may have to seek remedies for Your violations of this Public License. + +c. For the avoidance of doubt, the Licensor may also offer the Licensed Material under separate terms or conditions or stop distributing the Licensed Material at any time; however, doing so will not terminate this Public License. + +d. Sections 1, 5, 6, 7, and 8 survive termination of this Public License. + +### Section 7 – Other Terms and Conditions. + +a. The Licensor shall not be bound by any additional or different terms or conditions communicated by You unless expressly agreed. + +b. Any arrangements, understandings, or agreements regarding the Licensed Material not stated herein are separate from and independent of the terms and conditions of this Public License. + +### Section 8 – Interpretation. + +a. For the avoidance of doubt, this Public License does not, and shall not be interpreted to, reduce, limit, restrict, or impose conditions on any use of the Licensed Material that could lawfully be made without permission under this Public License. + +b. To the extent possible, if any provision of this Public License is deemed unenforceable, it shall be automatically reformed to the minimum extent necessary to make it enforceable. If the provision cannot be reformed, it shall be severed from this Public License without affecting the enforceability of the remaining terms and conditions. + +c. No term or condition of this Public License will be waived and no failure to comply consented to unless expressly agreed to by the Licensor. + +d. Nothing in this Public License constitutes or may be interpreted as a limitation upon, or waiver of, any privileges and immunities that apply to the Licensor or You, including from the legal processes of any jurisdiction or authority. + +``` +Creative Commons is not a party to its public licenses. Notwithstanding, Creative Commons may elect to apply one of its public licenses to material it publishes and in those instances will be considered the “Licensor.” Except for the limited purpose of indicating that material is shared under a Creative Commons public license or as otherwise permitted by the Creative Commons policies published at [creativecommons.org/policies](http://creativecommons.org/policies), Creative Commons does not authorize the use of the trademark “Creative Commons” or any other trademark or logo of Creative Commons without its prior written consent including, without limitation, in connection with any unauthorized modifications to any of its public licenses or any other arrangements, understandings, or agreements concerning use of licensed material. For the avoidance of doubt, this paragraph does not form part of the public licenses. + +Creative Commons may be contacted at creativecommons.org +``` From e2f6521525057c759493298dc25236b5f4e8b3f6 Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Sat, 25 Feb 2017 16:04:42 -0500 Subject: [PATCH 033/216] Dual license code and data under CC0 CC0 1.0 license text from https://github.com/greenelab/deep-review/pull/257/commits/86f159e620a212d9c3b023887c75515ce976be9b. --- LICENSE-CC0.md | 43 +++++++++++++++++++++++++++++++++++++++++++ README.md | 24 ++++++++++++++++++++++++ 2 files changed, 67 insertions(+) create mode 100644 LICENSE-CC0.md diff --git a/LICENSE-CC0.md b/LICENSE-CC0.md new file mode 100644 index 0000000..0490b5a --- /dev/null +++ b/LICENSE-CC0.md @@ -0,0 +1,43 @@ +# CC0 1.0 Universal + +``` +CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED HEREUNDER. +``` + +### Statement of Purpose + +The laws of most jurisdictions throughout the world automatically confer exclusive Copyright and Related Rights (defined below) upon the creator and subsequent owner(s) (each and all, an "owner") of an original work of authorship and/or a database (each, a "Work"). + +Certain owners wish to permanently relinquish those rights to a Work for the purpose of contributing to a commons of creative, cultural and scientific works ("Commons") that the public can reliably and without fear of later claims of infringement build upon, modify, incorporate in other works, reuse and redistribute as freely as possible in any form whatsoever and for any purposes, including without limitation commercial purposes. These owners may contribute to the Commons to promote the ideal of a free culture and the further production of creative, cultural and scientific works, or to gain reputation or greater distribution for their Work in part through the use and efforts of others. + +For these and/or other purposes and motivations, and without any expectation of additional consideration or compensation, the person associating CC0 with a Work (the "Affirmer"), to the extent that he or she is an owner of Copyright and Related Rights in the Work, voluntarily elects to apply CC0 to the Work and publicly distribute the Work under its terms, with knowledge of his or her Copyright and Related Rights in the Work and the meaning and intended legal effect of CC0 on those rights. + +1. __Copyright and Related Rights.__ A Work made available under CC0 may be protected by copyright and related or neighboring rights ("Copyright and Related Rights"). Copyright and Related Rights include, but are not limited to, the following: + + i. the right to reproduce, adapt, distribute, perform, display, communicate, and translate a Work; + + ii. moral rights retained by the original author(s) and/or performer(s); + + iii. publicity and privacy rights pertaining to a person's image or likeness depicted in a Work; + + iv. rights protecting against unfair competition in regards to a Work, subject to the limitations in paragraph 4(a), below; + + v. rights protecting the extraction, dissemination, use and reuse of data in a Work; + + vi. database rights (such as those arising under Directive 96/9/EC of the European Parliament and of the Council of 11 March 1996 on the legal protection of databases, and under any national implementation thereof, including any amended or successor version of such directive); and + + vii. other similar, equivalent or corresponding rights throughout the world based on applicable law or treaty, and any national implementations thereof. + +2. __Waiver.__ To the greatest extent permitted by, but not in contravention of, applicable law, Affirmer hereby overtly, fully, permanently, irrevocably and unconditionally waives, abandons, and surrenders all of Affirmer's Copyright and Related Rights and associated claims and causes of action, whether now known or unknown (including existing as well as future claims and causes of action), in the Work (i) in all territories worldwide, (ii) for the maximum duration provided by applicable law or treaty (including future time extensions), (iii) in any current or future medium and for any number of copies, and (iv) for any purpose whatsoever, including without limitation commercial, advertising or promotional purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each member of the public at large and to the detriment of Affirmer's heirs and successors, fully intending that such Waiver shall not be subject to revocation, rescission, cancellation, termination, or any other legal or equitable action to disrupt the quiet enjoyment of the Work by the public as contemplated by Affirmer's express Statement of Purpose. + +3. __Public License Fallback.__ Should any part of the Waiver for any reason be judged legally invalid or ineffective under applicable law, then the Waiver shall be preserved to the maximum extent permitted taking into account Affirmer's express Statement of Purpose. In addition, to the extent the Waiver is so judged Affirmer hereby grants to each affected person a royalty-free, non transferable, non sublicensable, non exclusive, irrevocable and unconditional license to exercise Affirmer's Copyright and Related Rights in the Work (i) in all territories worldwide, (ii) for the maximum duration provided by applicable law or treaty (including future time extensions), (iii) in any current or future medium and for any number of copies, and (iv) for any purpose whatsoever, including without limitation commercial, advertising or promotional purposes (the "License"). The License shall be deemed effective as of the date CC0 was applied by Affirmer to the Work. Should any part of the License for any reason be judged legally invalid or ineffective under applicable law, such partial invalidity or ineffectiveness shall not invalidate the remainder of the License, and in such case Affirmer hereby affirms that he or she will not (i) exercise any of his or her remaining Copyright and Related Rights in the Work or (ii) assert any associated claims and causes of action with respect to the Work, in either case contrary to Affirmer's express Statement of Purpose. + +4. __Limitations and Disclaimers.__ + + a. No trademark or patent rights held by Affirmer are waived, abandoned, surrendered, licensed or otherwise affected by this document. + + b. Affirmer offers the Work as-is and makes no representations or warranties of any kind concerning the Work, express, implied, statutory or otherwise, including without limitation warranties of title, merchantability, fitness for a particular purpose, non infringement, or the absence of latent or other defects, accuracy, or the present or absence of errors, whether or not discoverable, all to the greatest extent permissible under applicable law. + + c. Affirmer disclaims responsibility for clearing rights of other persons that may apply to the Work or any use thereof, including without limitation any person's Copyright and Related Rights in the Work. Further, Affirmer disclaims responsibility for obtaining any necessary consents, permissions or other rights required for any use of the Work. + + d. Affirmer understands and acknowledges that Creative Commons is not a party to this document and has no duty or obligation with respect to this CC0 or use of the Work. diff --git a/README.md b/README.md index 0813945..cb0b379 100644 --- a/README.md +++ b/README.md @@ -85,3 +85,27 @@ review will cover. [![Build Status](https://travis-ci.org/greenelab/deep-review.svg?branch=master)](https://travis-ci.org/greenelab/deep-review) [`.travis.yml`](.travis.yml) configures the continuous integration. + +## License + +This entirety of this repository is licensed under a CC BY 4.0 License ([`LICENSE.md`](LICENSE.md)), which allows reuse with attribution. +Please attribute by linking to https://github.com/greenelab/deep-review. + +Since CC BY is not ideal for code and data, certain repository components are also released under the CC0 1.0 public domain dedication ([`LICENSE-CC0.md`](LICENSE-CC0.md)). +All files matched by the following blog patterns are dual licensed under CC BY 4.0 and CC0 1.0: + ++ `*.sh` ++ `*.py` ++ `*.yml` ++ `*.json` ++ `*.bib` ++ `*.tsv` ++ `.gitignore` + +All other files are only available under CC BY 4.0, including: + ++ `*.md` ++ `*.html` ++ `*.pdf` + +Please open [an issue](https://github.com/greenelab/deep-review/issues) for any question related to licensing. From 1a9dfe05eb9dbd518f764a3a49e52b1cc13b1918 Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Sat, 25 Feb 2017 16:21:42 -0500 Subject: [PATCH 034/216] Cache generated citation metadata --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 49a3172..38b045d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,3 +13,6 @@ install: - source activate deepreview script: - sh build/build.sh +cache: + directories: + - references/generated From 1fef464d9771fda5233d7d515447c8c2a390c024 Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Sat, 25 Feb 2017 17:47:05 -0500 Subject: [PATCH 035/216] Add GitHub SSH deploy key --- ci/.gitignore | 2 ++ ci/README.md | 21 +++++++++++++++++++++ ci/deploy.key.enc | Bin 0 -> 3248 bytes ci/deploy.key.pub | 1 + 4 files changed, 24 insertions(+) create mode 100644 ci/.gitignore create mode 100644 ci/README.md create mode 100644 ci/deploy.key.enc create mode 100644 ci/deploy.key.pub diff --git a/ci/.gitignore b/ci/.gitignore new file mode 100644 index 0000000..8dd8cd8 --- /dev/null +++ b/ci/.gitignore @@ -0,0 +1,2 @@ +# SSH private key +deploy.key diff --git a/ci/README.md b/ci/README.md new file mode 100644 index 0000000..198bc0f --- /dev/null +++ b/ci/README.md @@ -0,0 +1,21 @@ +# Continuous integration / analysis directory + +[![Build Status](https://travis-ci.org/greenelab/deep-review.svg?branch=master)](https://travis-ci.org/greenelab/deep-review) + +This repository uses [continuous analysis](https://doi.org/10.1101/056473 "Reproducible Computational Workflows with Continuous Analysis") to create the manuscript and commit it back to GitHub. + +## Generating the GitHub deploy key + +To generate and configure the GitHub SSH deploy key, run the following commands from this directory. + +```sh +# Generate a new SSH key +ssh-keygen -t rsa -b 4096 -C "travis@travis-ci.com" -f deploy.sh +# Add deploy.key.pub to GitHub under the repository's settings +# see https://developer.github.com/guides/managing-deploy-keys/#deploy-keys + +# Encrypt and upload key to Travis CI. +# See https://docs.travis-ci.com/user/encrypting-files/ +travis encrypt-file deploy.key +# Modify the returned command to decrypt the key in ci/deploy.sh +``` diff --git a/ci/deploy.key.enc b/ci/deploy.key.enc new file mode 100644 index 0000000000000000000000000000000000000000..81ca3eacb0e860dc5b3b453a14caff56c8dc930d GIT binary patch literal 3248 zcmV;h3{Ug4CrAUD)qKWWNkzH3#ThiE2Y}Gjg@f`}+r%P+-r?|~>YltXOd9ayLAD-d$^obt?)@B}+=w6yGTgWFXX5w(c2k}v z-s;szEEy!j8JTlkTISz#xExneT~0#I9OsO|NJ{T$nxx5sK`5M&7p(R23VNTh6Wz?+ zeEW&R^A#Kmv!MH3_6MNK-Gj%*=n^SGIqkqih~1@a3}bB+9*@q#X}o5{1Rsg&`SS;x zU{SeMpnEmbk_Z+>t*+zwMzT^1U*-uBa5Lf)l1~U8`C&u>bexK-Nn0*#;%DiFCFQT( z$uSm#Ki5FnPpkU&?wlrrd_3>QMKd z&P9)9RYt!f;hZ+1)a{wpAV&0!yOOqL)dj_laFu4-vIZJp-;>n2BfTG zdSu^qbA(T}o9uUSvJv!xqaQTlF{-5b9Fq8%-_5IQc^cu`(|<~t`~*me34xRzh_TUP zNe5sZtef(`C72H$v*{S2w0$>ZrLC_5O;J61!k&Q(DA4PZpU9%YV>mPc$ILgyvC*5XoS09Y^JV85|Y{s=nxGypQ(rVO4!7DXlWyyeiT5z99Bdx+3IxS7#H{VS7<5`BQ+F?s8~yTgz~ViWmCm z@yg;kF4ie&`a3>VLgOfidDM=7qw^oZgQ@T(_=jnlL?&VVLwwkQ)KZPMb?z-=uC5iyB33HE6+xN^P=k6BLalN=u}WQ#+3M%SxjF!kd-gz}5y z|J1oD)2%uaR}X5ov+=;lpXJUih~bI=$92p(XX6K38L*aMh!Jb3oi1Ep<4)M7bHH={ zlMOOzWM~Tu$6@)gTC$QMmMlplw{={+4F?xEPZ|X+7@ZY$G`c6eA(U2LSDnp37Cdh4nF_ZSKn;cdklb64t z;ec`+N~d`5aeLLXmZEWXR_@P4ZS0W1!s9d3=4qJWDAT>mkzwlaD3DYp(r`@>o{9HI zeKijg(h70o%6v8fjiEQBKG{E0{8iY2nnk*^Wt-9S-c_qAy#(Nx!GXkL_QVQKgw58- zxr9)Q9MP>3gA;dXy zNm5IkpJ26C1);vz-4q^*Uc^L0oZ<;kcz82cvAA>YXie5`) zukL*Qao*MbkN?LhChBd*W83@G=SbB^O;;SD<)Ew6@s|Y&RJ+GWM@}ZfIrgu{CpGsy zI9N2-L-3ktl#@=UlYD7YHr;{1y6#VzR)<_~y-qK#c&;7>)SxFqU^q(9ElI@yQzUe&MbMdo@4Wmy_BC|~v9q$2L6jeVtXg{> zxK{UC>{P08xt1`b+m6gbhL;QJRT?DS!r{#GtI%LyU~YJ8rBP~tw6J(w`34cGER&== zLX#wUo$mnlU+o3AC@t-%(OTj&z5XII^tw$xZZChO>G|oGStvtip7i}6b{nHEX?@Xt zLC)AX=FmqnP-47VP;HX_nqUdji;66Hb4{N&f}fWQiH3OE(%#u2w?ds;-I}6p7*%)5 znpm6Xnt?B|Fu4Xdl<$lG-^Ygbr!bbn)mpzM^tv;`*?xTLMmc%gLeT`hf^YnhXY~A} zL~R8I80$jdtZ#zq^}AFe9g0SYnL&9Aw`za@;O6_u*q)cO#8LBEsKF#|xGlR;mtnIY zVIg0%Dn%PFGvd0}U5($9UDEx*FDC?vJ|oG%eOi;Uk_RCSk2$o=aTIR|NN%z=>4KtL zB#9dC_%^1{liRLw^@PlyI-f&mbK&>*B@P2{@p3>G$Akv=9Sk*~jVRO%z$VrM!R=H! z>&iURT&vyfA7sEhHmo=o1x+m(d*mq{B!Y~Qlxz->c#5Rh2B{)dJ~c*W=&+6N z16xVpRKJGD=v*kU-hb?NCSO5vA=i~R*t{>asZ*B?LC{fq)IA|=n7V;`rx{Q^lF`Nf zz}bvBhve1OaZ#C!;MY>PlK&~*j!F?VZOx$0Hg{v~_YzP(`+HZsGZ76_d{snqP1@d$ z)7^)15gnL$5VU88mn{dsvl8x?^!iZOq>Qks{yA_lToWe<>>m!OF$tZIGkRECTkQ8Y zFSlbRVl?NJSm_zGj9XZ2PQ*Jh69h6^w;76C0tH9pq&GgVGvf-7+h9Q4CVYYkOy}+Q z%+D`ng)F-P)~9tJu9ty>Rl$K>z*4aQX9B0GraF-hINRLFTq77it8?3tVjv9j#n4qV zTiW>(d(P%bvA-Y=OW05Vn0gq{09aJ??LX($UnGEs`UH}osv=TJ?R!J%`w|THul1d^ z4+ZQ2vDtdRyL;V0F2OF{j5}WdU5o>T}Wrw)8e2GLH1f zf6l$iU`Z^`QZ!*Cn`eY$B|;aW5J-JgB*f=zbU z$4QzSa?-<97Bnp5gSJ9aK@^vjZ(Cv`j$zi_N~VlANF~o>{0?H(kpY(>ukGhai#4Qa z{u0lxP@qht*ga-&su3YQUY>J6=F3YeRrG|`m>|EAG`xA?06kD9;_^0<(xoBFWxO8P zd)pl;d0|;T6Mo&>oXSE8ggh5r4ykgm-gtj>evb!P5rK2_VYC8e$jdgQu{Pv>XO;CQ z5={Nrd+DmI{F=5-)Zh5G^6k*hoz2n%kJGW(RuLmOl=kU%)y_+!A1Pxw+aSStwC*qc z+RPVLN>GLKO;qauEJ|)wNr5ctAciE| zW~w3}wx%GDBr9W{aj0V+5(CfF?0vw;{_7mUQt}u1mvzg4eXJ_H9o`2)kGsJw*@xNt z;)|%c-tBr?jLSjzl(7Hmp3is6G81bOUyWnDn?UdWDPEmElogcSLX!ZnOfUk4l5Y5> z;4pE5$B#@8aVggvYL2$HSmv>NPelfbBT4l2DX+#ASL0crSRkN6H7BSjrMXdX_Zy$R z7lxmz9^K${w(ovd-*(Z@8}wrW*X#y}2Sf5`r2;r(GT>2)WgXi*FtA&kId?OU98O)p z&Jv;J;7x;J_BShiFT9m%{Jz8U&iAwN6&>lLv_Q>FnM#?8y70upFQTAMolF_X>U7;X z#tH)+;tVli>BGBkZL(0L;h}yXqns>!`cO*QpCM}=mkUH_lbYh$13dFlPS&~yaeNqw zW1aZ0Dm8+_vQylq+6u8@S(~EzPI7h`ub%QiZRbjDAy9}LsMqCoDM#=R!xLCepd>0* zpTc~w+a%dN-zb17^lJ<~3y_!)6k z)iU!*c^WrU>p6c&nyz$b=TS@_1AR!xCU=<{c)BlkPPsgD!$ z%z|lIj7X2`9QJoQ2M6CwEUT{REtHbY4q_xlp}1xb2bmKcxZfFbNG;Ay`k=|v;(X*1 iE>ePIzy|}0k0ehUV>NLc3tZC$**1)&av)IDPX<)HsZDhN literal 0 HcmV?d00001 diff --git a/ci/deploy.key.pub b/ci/deploy.key.pub new file mode 100644 index 0000000..719e746 --- /dev/null +++ b/ci/deploy.key.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC8XiVsudPG/YBgkKuqpaez+LGH8+86jNtID0/FpVJVtiqlvVRG6YO7YtJZJc8DNoO0epClt3lMpwu/ISPvixvPXG5OOiUW9ALZPoDx4loN1SY3+ifA2LyTa5lFYtrijJvqtm9jZkzEG8B/nUgtDey6ROCFnNAYASVb1INDHsZznlhZF1Z+AckIAT7dgZ0GZZ8GVKLaSfLSfQ2cQtSXVRxgWqxs6ItRskUJD4T5Ao6brVec2SsUT+xm4eZvl4sZCqG/vNklqvuhs1wJ1ordmehLyj82D3Yy6079dRsR2AULTkPDOSo3HhoHZ3wD+95FtAyAku25pXWZxzY8Yf21I7iidzOP2bBUumlIeo/EZCirU1APOJy1bWyioqznIyOeBFFWQ2oB7rL64XfXJFhfwMeB6aJlD2iiNaw2Hm/zmsR6uwEwctiOnvD5JzEGV0wCfSSvX6jDJ5cFi2DB92B/XjwKGahG5fsbKpu5RHbfv0BBRoniR1Hd5ntu7uSYrNwcCXnEfiHdazThLAqaRXg/N10IQoUkdpzUTez2ejYD6R+1xOShYOF/CKFpkYOMM92m2n8DkB1kodPKJCe6kX3/eGfr/Y+q/NIjUQKy2TSz6HHPn7vc+N7DX5DCQHGgd/bMAeIfKzybNa3GHmLMbyMU1572nbLXgScc32xmb5ekq16FVw== travis@travis-ci.com From 1a0b9c7629835db427ffa138c260251c803f02c2 Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Sat, 25 Feb 2017 18:16:12 -0500 Subject: [PATCH 036/216] Wrap long line in .travis.yml --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 38b045d..495aa63 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,8 @@ language: generic before_install: - sudo apt-get update --quiet --quiet - - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh --output-document miniconda.sh + - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh + --output-document miniconda.sh - bash miniconda.sh -b -p $HOME/miniconda - export PATH="$HOME/miniconda/bin:$PATH" - hash -r From 736aedbb9c2dd08165fc4c5b9345ae464657a688 Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Sat, 25 Feb 2017 18:24:47 -0500 Subject: [PATCH 037/216] Add deployment script --- .travis.yml | 4 ++++ ci/README.md | 3 +++ ci/deploy.sh | 42 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+) create mode 100644 ci/deploy.sh diff --git a/.travis.yml b/.travis.yml index 495aa63..5e45b1a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,3 +17,7 @@ script: cache: directories: - references/generated +after_success: + - test $TRAVIS_BRANCH = "master" && + test $TRAVIS_PULL_REQUEST = "false" && + sh ci/deploy.sh diff --git a/ci/README.md b/ci/README.md index 198bc0f..db62f51 100644 --- a/ci/README.md +++ b/ci/README.md @@ -3,6 +3,9 @@ [![Build Status](https://travis-ci.org/greenelab/deep-review.svg?branch=master)](https://travis-ci.org/greenelab/deep-review) This repository uses [continuous analysis](https://doi.org/10.1101/056473 "Reproducible Computational Workflows with Continuous Analysis") to create the manuscript and commit it back to GitHub. +[`deploy.sh`](deploy.sh) runs on successful `master` branch builds that are not pull requests. +The contents of `/output` are committed to the `gh-pages` branch. +The contents of `/references/generated` are committed to the `references` branch. ## Generating the GitHub deploy key diff --git a/ci/deploy.sh b/ci/deploy.sh new file mode 100644 index 0000000..53fdb34 --- /dev/null +++ b/ci/deploy.sh @@ -0,0 +1,42 @@ +set -o errexit + +# Configure git +git config --global push.default simple +git config --global user.email "travis@travis-ci.com" +git config --global user.name "Travis CI" +git checkout $TRAVIS_BRANCH +git remote set-url origin git@github.com:greenelab/deep-review.git + +# Decrypt and add SSH key +openssl aes-256-cbc \ + -K $encrypted_0dd7e5f24ac8_key \ + -iv $encrypted_0dd7e5f24ac8_iv \ + -in ci/deploy.key.enc \ + -out ci/deploy.key -d +eval `ssh-agent -s` +chmod 600 ci/deploy.key +ssh-add ci/deploy.key + +# Commit message +message="\ +$TRAVIS_COMMIT_MESSAGE + +*** + +The remainder of the commit message has been auto-generated. + +TRAVIS_COMMIT=$TRAVIS_COMMIT +TRAVIS_COMMIT_RANGE=$TRAVIS_COMMIT_RANGE + +Created by Travis CI build number $TRAVIS_BUILD_NUMBER and job number $TRAVIS_JOB_NUMBER. +https://travis-ci.org/greenelab/deep-review/builds/$TRAVIS_BUILD_ID +https://travis-ci.org/greenelab/deep-review/jobs/$TRAVIS_JOB_ID + +Committed on `date --iso-8601=seconds --universal`. +" + +# Deploy the reference data to references +ghp-import -p -b references -m $MESSAGE references/generated + +# Deploy the output to gh-pages +ghp-import -p -b gh-pages -m $MESSAGE output From de72db1254ea39120f40eda03d1240a9f42bf4ab Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Sat, 25 Feb 2017 18:37:41 -0500 Subject: [PATCH 038/216] Attempt to fix ghp-import error Error is: ``` ghp-import: error: No import directory specified. ``` Testing "double quotes" and 'single quotes' --- ci/deploy.sh | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/ci/deploy.sh b/ci/deploy.sh index 53fdb34..52581fa 100644 --- a/ci/deploy.sh +++ b/ci/deploy.sh @@ -18,25 +18,26 @@ chmod 600 ci/deploy.key ssh-add ci/deploy.key # Commit message -message="\ +MESSAGE="\ $TRAVIS_COMMIT_MESSAGE *** -The remainder of the commit message has been auto-generated. - -TRAVIS_COMMIT=$TRAVIS_COMMIT -TRAVIS_COMMIT_RANGE=$TRAVIS_COMMIT_RANGE +This build was triggered by +https://github.com/greenelab/deep-review/commit/$TRAVIS_COMMIT Created by Travis CI build number $TRAVIS_BUILD_NUMBER and job number $TRAVIS_JOB_NUMBER. https://travis-ci.org/greenelab/deep-review/builds/$TRAVIS_BUILD_ID https://travis-ci.org/greenelab/deep-review/jobs/$TRAVIS_JOB_ID Committed on `date --iso-8601=seconds --universal`. + +[ci skip] " +echo "$MESSAGE" # Deploy the reference data to references -ghp-import -p -b references -m $MESSAGE references/generated +ghp-import -p -b references -m "$MESSAGE" references/generated # Deploy the output to gh-pages -ghp-import -p -b gh-pages -m $MESSAGE output +ghp-import -p -b gh-pages -m "$MESSAGE" output From da704b69e52141321cd41f5a6151631eb4da56e6 Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Sat, 25 Feb 2017 19:56:33 -0500 Subject: [PATCH 039/216] CI: fetch gh-pages and references branches --- ci/deploy.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ci/deploy.sh b/ci/deploy.sh index 52581fa..5ed5f0f 100644 --- a/ci/deploy.sh +++ b/ci/deploy.sh @@ -17,6 +17,11 @@ eval `ssh-agent -s` chmod 600 ci/deploy.key ssh-add ci/deploy.key +# Fetch and create gh-pages and references branches +# Travis does a shallow and single branch git clone +git remote set-branches --add origin gh-pages references +git fetch origin gh-pages:gh-pages references:references + # Commit message MESSAGE="\ $TRAVIS_COMMIT_MESSAGE @@ -34,7 +39,6 @@ Committed on `date --iso-8601=seconds --universal`. [ci skip] " -echo "$MESSAGE" # Deploy the reference data to references ghp-import -p -b references -m "$MESSAGE" references/generated From cf190884c00e8dd50f8d1fb3a4b0b66bb15230bb Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Sat, 25 Feb 2017 20:56:42 -0500 Subject: [PATCH 040/216] CI: transmit commit author details Modify auto-generated commit message --- ci/deploy.sh | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/ci/deploy.sh b/ci/deploy.sh index 5ed5f0f..0ea20f9 100644 --- a/ci/deploy.sh +++ b/ci/deploy.sh @@ -2,8 +2,8 @@ set -o errexit # Configure git git config --global push.default simple -git config --global user.email "travis@travis-ci.com" -git config --global user.name "Travis CI" +git config --global user.email `git log --max-count=1 --format='%ae'` +git config --global user.name `git log --max-count=1 --format='%an'` git checkout $TRAVIS_BRANCH git remote set-url origin git@github.com:greenelab/deep-review.git @@ -24,20 +24,20 @@ git fetch origin gh-pages:gh-pages references:references # Commit message MESSAGE="\ -$TRAVIS_COMMIT_MESSAGE - -*** +`git log --max-count=1 --format='%s'` -This build was triggered by -https://github.com/greenelab/deep-review/commit/$TRAVIS_COMMIT +This build is based on +https://github.com/greenelab/deep-review/commit/$TRAVIS_COMMIT. -Created by Travis CI build number $TRAVIS_BUILD_NUMBER and job number $TRAVIS_JOB_NUMBER. +This commit was created by the following Travis CI build and job: https://travis-ci.org/greenelab/deep-review/builds/$TRAVIS_BUILD_ID https://travis-ci.org/greenelab/deep-review/jobs/$TRAVIS_JOB_ID -Committed on `date --iso-8601=seconds --universal`. - [ci skip] + +The full commit message that triggered this build is copied below: + +$TRAVIS_COMMIT_MESSAGE " # Deploy the reference data to references From 0645cad7faf8c015e2a96a4fbbe1a8a0d8b0d8eb Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Sun, 26 Feb 2017 13:46:33 -0500 Subject: [PATCH 041/216] Use long arguments with ghp-import As as result of https://github.com/davisp/ghp-import/pull/62. --- build/environment.yml | 2 +- ci/deploy.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/build/environment.yml b/build/environment.yml index 8c90e38..ff69c11 100644 --- a/build/environment.yml +++ b/build/environment.yml @@ -11,4 +11,4 @@ dependencies: - pip: - arxiv2bib==1.0.7 - bibtexparser==0.6.2 - - ghp-import==0.5.4 + - ghp-import==0.5.5 diff --git a/ci/deploy.sh b/ci/deploy.sh index 0ea20f9..e812c34 100644 --- a/ci/deploy.sh +++ b/ci/deploy.sh @@ -41,7 +41,7 @@ $TRAVIS_COMMIT_MESSAGE " # Deploy the reference data to references -ghp-import -p -b references -m "$MESSAGE" references/generated +ghp-import --push --branch=references --message="$MESSAGE" references/generated # Deploy the output to gh-pages -ghp-import -p -b gh-pages -m "$MESSAGE" output +ghp-import --push --branch=gh-pages --message="$MESSAGE" output From b5ffb9edfc81f3d99a1b712eafe107edeebd2e64 Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Sun, 26 Feb 2017 14:21:41 -0500 Subject: [PATCH 042/216] Document references/generated directory --- .gitignore | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 8e13d34..9b5c29a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,9 @@ -# Output files +# Generated manuscript output files output -references/generated + +# Generated reference files +references/generated/* +!references/generated/README.md # Python __pycache__/ From 9ba02f687ca3546d4289a55414e45109a27cc9ac Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Sun, 26 Feb 2017 15:13:10 -0500 Subject: [PATCH 043/216] Build: print bracketed text that's not refs --- build/citations.py | 9 +++++++++ build/references.py | 5 +++++ 2 files changed, 14 insertions(+) diff --git a/build/citations.py b/build/citations.py index f5e3b99..3845740 100644 --- a/build/citations.py +++ b/build/citations.py @@ -32,6 +32,15 @@ def get_references_from_text(text): return refs +def get_brackets_without_reference(text): + """ + Find bracketed text that does not start with @. Does not match + brackets that are followed by an open parenthesis. + """ + pattern = re.compile(r'(\[[^@].*?\])[^(]', flags=re.DOTALL) + return list(pattern.findall(text)) + + def get_text(directory): """ Return a dictionary of section texts in the specified directory. diff --git a/build/references.py b/build/references.py index 2d20e64..f664589 100644 --- a/build/references.py +++ b/build/references.py @@ -16,6 +16,7 @@ from citations import ( citation_to_metadata, citeproc_passthrough, + get_brackets_without_reference, get_references_from_text, get_text, validate_reference, @@ -32,6 +33,10 @@ text = get_text('../sections') refs = sorted(get_references_from_text(text)) +warn_refs = get_brackets_without_reference(text) +if warn_refs: + print('WARNING: The following bracketed texts are not references:') + print('\n'.join(warn_refs)) bad_refs = list(filter(None, map(validate_reference, refs))) if bad_refs: print('\n'.join(bad_refs)) From 7c97c64552caff9556950fff3b3cf37b374540d3 Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Sun, 26 Feb 2017 16:46:59 -0500 Subject: [PATCH 044/216] Build only master branch --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 5e45b1a..eca4d51 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,7 @@ language: generic +branches: + only: + - master before_install: - sudo apt-get update --quiet --quiet - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh From 0a4f2d8924b49805fc32171470197fda70e2946f Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Sun, 26 Feb 2017 16:49:49 -0500 Subject: [PATCH 045/216] Fix regex error --- build/citations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/citations.py b/build/citations.py index 3845740..e8b0071 100644 --- a/build/citations.py +++ b/build/citations.py @@ -37,7 +37,7 @@ def get_brackets_without_reference(text): Find bracketed text that does not start with @. Does not match brackets that are followed by an open parenthesis. """ - pattern = re.compile(r'(\[[^@].*?\])[^(]', flags=re.DOTALL) + pattern = re.compile(r'(\[[^@][^\]]*?\])[^(]', flags=re.DOTALL) return list(pattern.findall(text)) From ce520defbbb86038397e15c9eedcf443f494f52a Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Sun, 26 Feb 2017 18:46:25 -0500 Subject: [PATCH 046/216] Add CSS style to HTML pandoc output (#263) From https://gist.github.com/dashed/6714393 by @dashed. MIT Licensed. Versioned link https://git.io/vykxo. --- .gitignore | 3 +- build/build.sh | 1 + output/github-pandoc.css | 424 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 427 insertions(+), 1 deletion(-) create mode 100644 output/github-pandoc.css diff --git a/.gitignore b/.gitignore index 9b5c29a..0c962c3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # Generated manuscript output files -output +output/index.html +output/deep-review.pdf # Generated reference files references/generated/* diff --git a/build/build.sh b/build/build.sh index 858d751..12745c8 100644 --- a/build/build.sh +++ b/build/build.sh @@ -19,6 +19,7 @@ pandoc --verbose \ --from=markdown --to=html \ --bibliography=$BIBLIOGRAPHY_PATH \ --csl=$CSL_PATH \ + --css=github-pandoc.css \ --output=output/index.html \ $INPUT_PATH diff --git a/output/github-pandoc.css b/output/github-pandoc.css new file mode 100644 index 0000000..0607023 --- /dev/null +++ b/output/github-pandoc.css @@ -0,0 +1,424 @@ +/*! normalize.css v2.1.3 | MIT License | git.io/normalize */ + +/* ========================================================================== + HTML5 display definitions + ========================================================================== */ + +/** + * Correct `block` display not defined in IE 8/9. + */ + +article, +aside, +details, +figcaption, +figure, +footer, +header, +hgroup, +main, +nav, +section, +summary { + display: block; +} + +/** + * Correct `inline-block` display not defined in IE 8/9. + */ + +audio, +canvas, +video { + display: inline-block; +} + +/** + * Prevent modern browsers from displaying `audio` without controls. + * Remove excess height in iOS 5 devices. + */ + +audio:not([controls]) { + display: none; + height: 0; +} + +/** + * Address `[hidden]` styling not present in IE 8/9. + * Hide the `template` element in IE, Safari, and Firefox < 22. + */ + +[hidden], +template { + display: none; +} + +/* ========================================================================== + Base + ========================================================================== */ + +/** + * 1. Set default font family to sans-serif. + * 2. Prevent iOS text size adjust after orientation change, without disabling + * user zoom. + */ + +html { + font-family: sans-serif; /* 1 */ + -ms-text-size-adjust: 100%; /* 2 */ + -webkit-text-size-adjust: 100%; /* 2 */ +} + +/** + * Remove default margin. + */ + +body { + margin: 0; +} + +/* ========================================================================== + Links + ========================================================================== */ + +/** + * Remove the gray background color from active links in IE 10. + */ + +a { + background: transparent; +} + +/** + * Address `outline` inconsistency between Chrome and other browsers. + */ + +a:focus { + outline: thin dotted; +} + +/** + * Improve readability when focused and also mouse hovered in all browsers. + */ + +a:active, +a:hover { + outline: 0; +} + +/* ========================================================================== + Typography + ========================================================================== */ + +/** + * Address variable `h1` font-size and margin within `section` and `article` + * contexts in Firefox 4+, Safari 5, and Chrome. + */ + +h1 { + font-size: 2em; + margin: 0.67em 0; +} + +/** + * Address styling not present in IE 8/9, Safari 5, and Chrome. + */ + +abbr[title] { + border-bottom: 1px dotted; +} + +/** + * Address style set to `bolder` in Firefox 4+, Safari 5, and Chrome. + */ + +b, +strong { + font-weight: bold; +} + +/** + * Address styling not present in Safari 5 and Chrome. + */ + +dfn { + font-style: italic; +} + +/** + * Address differences between Firefox and other browsers. + */ + +hr { + -moz-box-sizing: content-box; + box-sizing: content-box; + height: 0; +} + +/** + * Address styling not present in IE 8/9. + */ + +mark { + background: #ff0; + color: #000; +} + +/** + * Correct font family set oddly in Safari 5 and Chrome. + */ + +code, +kbd, +pre, +samp { + font-family: monospace, serif; + font-size: 1em; +} + +/** + * Improve readability of pre-formatted text in all browsers. + */ + +pre { + white-space: pre-wrap; +} + +/** + * Set consistent quote types. + */ + +q { + quotes: "\201C" "\201D" "\2018" "\2019"; +} + +/** + * Address inconsistent and variable font size in all browsers. + */ + +small { + font-size: 80%; +} + +/** + * Prevent `sub` and `sup` affecting `line-height` in all browsers. + */ + +sub, +sup { + font-size: 75%; + line-height: 0; + position: relative; + vertical-align: baseline; +} + +sup { + top: -0.5em; +} + +sub { + bottom: -0.25em; +} + +/* ========================================================================== + Embedded content + ========================================================================== */ + +/** + * Remove border when inside `a` element in IE 8/9. + */ + +img { + border: 0; +} + +/** + * Correct overflow displayed oddly in IE 9. + */ + +svg:not(:root) { + overflow: hidden; +} + +/* ========================================================================== + Figures + ========================================================================== */ + +/** + * Address margin not present in IE 8/9 and Safari 5. + */ + +figure { + margin: 0; +} + +/* ========================================================================== + Forms + ========================================================================== */ + +/** + * Define consistent border, margin, and padding. + */ + +fieldset { + border: 1px solid #c0c0c0; + margin: 0 2px; + padding: 0.35em 0.625em 0.75em; +} + +/** + * 1. Correct `color` not being inherited in IE 8/9. + * 2. Remove padding so people aren't caught out if they zero out fieldsets. + */ + +legend { + border: 0; /* 1 */ + padding: 0; /* 2 */ +} + +/** + * 1. Correct font family not being inherited in all browsers. + * 2. Correct font size not being inherited in all browsers. + * 3. Address margins set differently in Firefox 4+, Safari 5, and Chrome. + */ + +button, +input, +select, +textarea { + font-family: inherit; /* 1 */ + font-size: 100%; /* 2 */ + margin: 0; /* 3 */ +} + +/** + * Address Firefox 4+ setting `line-height` on `input` using `!important` in + * the UA stylesheet. + */ + +button, +input { + line-height: normal; +} + +/** + * Address inconsistent `text-transform` inheritance for `button` and `select`. + * All other form control elements do not inherit `text-transform` values. + * Correct `button` style inheritance in Chrome, Safari 5+, and IE 8+. + * Correct `select` style inheritance in Firefox 4+ and Opera. + */ + +button, +select { + text-transform: none; +} + +/** + * 1. Avoid the WebKit bug in Android 4.0.* where (2) destroys native `audio` + * and `video` controls. + * 2. Correct inability to style clickable `input` types in iOS. + * 3. Improve usability and consistency of cursor style between image-type + * `input` and others. + */ + +button, +html input[type="button"], /* 1 */ +input[type="reset"], +input[type="submit"] { + -webkit-appearance: button; /* 2 */ + cursor: pointer; /* 3 */ +} + +/** + * Re-set default cursor for disabled elements. + */ + +button[disabled], +html input[disabled] { + cursor: default; +} + +/** + * 1. Address box sizing set to `content-box` in IE 8/9/10. + * 2. Remove excess padding in IE 8/9/10. + */ + +input[type="checkbox"], +input[type="radio"] { + box-sizing: border-box; /* 1 */ + padding: 0; /* 2 */ +} + +/** + * 1. Address `appearance` set to `searchfield` in Safari 5 and Chrome. + * 2. Address `box-sizing` set to `border-box` in Safari 5 and Chrome + * (include `-moz` to future-proof). + */ + +input[type="search"] { + -webkit-appearance: textfield; /* 1 */ + -moz-box-sizing: content-box; + -webkit-box-sizing: content-box; /* 2 */ + box-sizing: content-box; +} + +/** + * Remove inner padding and search cancel button in Safari 5 and Chrome + * on OS X. + */ + +input[type="search"]::-webkit-search-cancel-button, +input[type="search"]::-webkit-search-decoration { + -webkit-appearance: none; +} + +/** + * Remove inner padding and border in Firefox 4+. + */ + +button::-moz-focus-inner, +input::-moz-focus-inner { + border: 0; + padding: 0; +} + +/** + * 1. Remove default vertical scrollbar in IE 8/9. + * 2. Improve readability and alignment in all browsers. + */ + +textarea { + overflow: auto; /* 1 */ + vertical-align: top; /* 2 */ +} + +/* ========================================================================== + Tables + ========================================================================== */ + +/** + * Remove most spacing between table cells. + */ + +table { + border-collapse: collapse; + border-spacing: 0; +} + +.go-top { +position: fixed; +bottom: 2em; +right: 2em; +text-decoration: none; +background-color: #E0E0E0; +font-size: 12px; +padding: 1em; +display: inline; +} + +/* Github css */ + +html,body{ margin: auto; + padding-right: 1em; + padding-left: 1em; + max-width: 44em; color:black;}*:not('#mkdbuttons'){margin:0;padding:0}body{font:13.34px helvetica,arial,freesans,clean,sans-serif;-webkit-font-smoothing:subpixel-antialiased;line-height:1.4;padding:3px;background:#fff;border-radius:3px;-moz-border-radius:3px;-webkit-border-radius:3px}p{margin:1em 0}a{color:#4183c4;text-decoration:none}body{background-color:#fff;padding:30px;margin:15px;font-size:14px;line-height:1.6}body>*:first-child{margin-top:0!important}body>*:last-child{margin-bottom:0!important}@media screen{body{box-shadow:0 0 0 1px #cacaca,0 0 0 4px #eee}}h1,h2,h3,h4,h5,h6{margin:20px 0 10px;padding:0;font-weight:bold;-webkit-font-smoothing:subpixel-antialiased;cursor:text}h1{font-size:28px;color:#000}h2{font-size:24px;border-bottom:1px solid #ccc;color:#000}h3{font-size:18px;color:#333}h4{font-size:16px;color:#333}h5{font-size:14px;color:#333}h6{color:#777;font-size:14px}p,blockquote,table,pre{margin:15px 0}ul{padding-left:30px}ol{padding-left:30px}ol li ul:first-of-type{margin-top:0}hr{background:transparent url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAYAAAAECAYAAACtBE5DAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAyJpVFh0WE1MOmNvbS5hZG9iZS54bXAAAAAAADw/eHBhY2tldCBiZWdpbj0i77u/IiBpZD0iVzVNME1wQ2VoaUh6cmVTek5UY3prYzlkIj8+IDx4OnhtcG1ldGEgeG1sbnM6eD0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IkFkb2JlIFhNUCBDb3JlIDUuMC1jMDYwIDYxLjEzNDc3NywgMjAxMC8wMi8xMi0xNzozMjowMCAgICAgICAgIj4gPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4gPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIgeG1sbnM6eG1wPSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvIiB4bWxuczp4bXBNTT0iaHR0cDovL25zLmFkb2JlLmNvbS94YXAvMS4wL21tLyIgeG1sbnM6c3RSZWY9Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC9zVHlwZS9SZXNvdXJjZVJlZiMiIHhtcDpDcmVhdG9yVG9vbD0iQWRvYmUgUGhvdG9zaG9wIENTNSBNYWNpbnRvc2giIHhtcE1NOkluc3RhbmNlSUQ9InhtcC5paWQ6OENDRjNBN0E2NTZBMTFFMEI3QjRBODM4NzJDMjlGNDgiIHhtcE1NOkRvY3VtZW50SUQ9InhtcC5kaWQ6OENDRjNBN0I2NTZBMTFFMEI3QjRBODM4NzJDMjlGNDgiPiA8eG1wTU06RGVyaXZlZEZyb20gc3RSZWY6aW5zdGFuY2VJRD0ieG1wLmlpZDo4Q0NGM0E3ODY1NkExMUUwQjdCNEE4Mzg3MkMyOUY0OCIgc3RSZWY6ZG9jdW1lbnRJRD0ieG1wLmRpZDo4Q0NGM0E3OTY1NkExMUUwQjdCNEE4Mzg3MkMyOUY0OCIvPiA8L3JkZjpEZXNjcmlwdGlvbj4gPC9yZGY6UkRGPiA8L3g6eG1wbWV0YT4gPD94cGFja2V0IGVuZD0iciI/PqqezsUAAAAfSURBVHjaYmRABcYwBiM2QSA4y4hNEKYDQxAEAAIMAHNGAzhkPOlYAAAAAElFTkSuQmCC) repeat-x 0 0;border:0 none;color:#ccc;height:4px;padding:0}body>h2:first-child{margin-top:0;padding-top:0}body>h1:first-child{margin-top:0;padding-top:0}body>h1:first-child+h2{margin-top:0;padding-top:0}body>h3:first-child,body>h4:first-child,body>h5:first-child,body>h6:first-child{margin-top:0;padding-top:0}a:first-child h1,a:first-child h2,a:first-child h3,a:first-child h4,a:first-child h5,a:first-child h6{margin-top:0;padding-top:0}h1+p,h2+p,h3+p,h4+p,h5+p,h6+p,ul li>:first-child,ol li>:first-child{margin-top:0}dl{padding:0}dl dt{font-size:14px;font-weight:bold;font-style:italic;padding:0;margin:15px 0 5px}dl dt:first-child{padding:0}dl dt>:first-child{margin-top:0}dl dt>:last-child{margin-bottom:0}dl dd{margin:0 0 15px;padding:0 15px}dl dd>:first-child{margin-top:0}dl dd>:last-child{margin-bottom:0}blockquote{border-left:4px solid #DDD;padding:0 15px;color:#777}blockquote>:first-child{margin-top:0}blockquote>:last-child{margin-bottom:0}table{border-collapse:collapse;border-spacing:0;font-size:100%;font:inherit}table th{font-weight:bold;border:1px solid #ccc;padding:6px 13px}table td{border:1px solid #ccc;padding:6px 13px}table tr{border-top:1px solid #ccc;background-color:#fff}table tr:nth-child(2n){background-color:#f8f8f8}img{max-width:100%}code,tt{margin:0 2px;padding:0 5px;white-space:nowrap;border:1px solid #eaeaea;background-color:#f8f8f8;border-radius:3px;font-family:Consolas,'Liberation Mono',Courier,monospace;font-size:12px;color:#333}pre>code{margin:0;padding:0;white-space:pre;border:0;background:transparent}.highlight pre{background-color:#f8f8f8;border:1px solid #ccc;font-size:13px;line-height:19px;overflow:auto;padding:6px 10px;border-radius:3px}pre{background-color:#f8f8f8;border:1px solid #ccc;font-size:13px;line-height:19px;overflow:auto;padding:6px 10px;border-radius:3px}pre code,pre tt{background-color:transparent;border:0}.poetry pre{font-family:Georgia,Garamond,serif!important;font-style:italic;font-size:110%!important;line-height:1.6em;display:block;margin-left:1em}.poetry pre code{font-family:Georgia,Garamond,serif!important;word-break:break-all;word-break:break-word;-webkit-hyphens:auto;-moz-hyphens:auto;hyphens:auto;white-space:pre-wrap}sup,sub,a.footnote{font-size:1.4ex;height:0;line-height:1;vertical-align:super;position:relative}sub{vertical-align:sub;top:-1px}@media print{body{background:#fff}img,pre,blockquote,table,figure{page-break-inside:avoid}body{background:#fff;border:0}code{background-color:#fff;color:#333!important;padding:0 .2em;border:1px solid #dedede}pre{background:#fff}pre code{background-color:white!important;overflow:visible}}@media screen{body.inverted{color:#eee!important;border-color:#555;box-shadow:none}.inverted body,.inverted hr .inverted p,.inverted td,.inverted li,.inverted h1,.inverted h2,.inverted h3,.inverted h4,.inverted h5,.inverted h6,.inverted th,.inverted .math,.inverted caption,.inverted dd,.inverted dt,.inverted blockquote{color:#eee!important;border-color:#555;box-shadow:none}.inverted td,.inverted th{background:#333}.inverted h2{border-color:#555}.inverted hr{border-color:#777;border-width:1px!important}::selection{background:rgba(157,193,200,0.5)}h1::selection{background-color:rgba(45,156,208,0.3)}h2::selection{background-color:rgba(90,182,224,0.3)}h3::selection,h4::selection,h5::selection,h6::selection,li::selection,ol::selection{background-color:rgba(133,201,232,0.3)}code::selection{background-color:rgba(0,0,0,0.7);color:#eee}code span::selection{background-color:rgba(0,0,0,0.7)!important;color:#eee!important}a::selection{background-color:rgba(255,230,102,0.2)}.inverted a::selection{background-color:rgba(255,230,102,0.6)}td::selection,th::selection,caption::selection{background-color:rgba(180,237,95,0.5)}.inverted{background:#0b2531;background:#252a2a}.inverted body{background:#252a2a}.inverted a{color:#acd1d5}}.highlight .c{color:#998;font-style:italic}.highlight .err{color:#a61717;background-color:#e3d2d2}.highlight .k,.highlight .o{font-weight:bold}.highlight .cm{color:#998;font-style:italic}.highlight .cp{color:#999;font-weight:bold}.highlight .c1{color:#998;font-style:italic}.highlight .cs{color:#999;font-weight:bold;font-style:italic}.highlight .gd{color:#000;background-color:#fdd}.highlight .gd .x{color:#000;background-color:#faa}.highlight .ge{font-style:italic}.highlight .gr{color:#a00}.highlight .gh{color:#999}.highlight .gi{color:#000;background-color:#dfd}.highlight .gi .x{color:#000;background-color:#afa}.highlight .go{color:#888}.highlight .gp{color:#555}.highlight .gs{font-weight:bold}.highlight .gu{color:#800080;font-weight:bold}.highlight .gt{color:#a00}.highlight .kc,.highlight .kd,.highlight .kn,.highlight .kp,.highlight .kr{font-weight:bold}.highlight .kt{color:#458;font-weight:bold}.highlight .m{color:#099}.highlight .s{color:#d14}.highlight .na{color:#008080}.highlight .nb{color:#0086b3}.highlight .nc{color:#458;font-weight:bold}.highlight .no{color:#008080}.highlight .ni{color:#800080}.highlight .ne,.highlight .nf{color:#900;font-weight:bold}.highlight .nn{color:#555}.highlight .nt{color:#000080}.highlight .nv{color:#008080}.highlight .ow{font-weight:bold}.highlight .w{color:#bbb}.highlight .mf,.highlight .mh,.highlight .mi,.highlight .mo{color:#099}.highlight .sb,.highlight .sc,.highlight .sd,.highlight .s2,.highlight .se,.highlight .sh,.highlight .si,.highlight .sx{color:#d14}.highlight .sr{color:#009926}.highlight .s1{color:#d14}.highlight .ss{color:#990073}.highlight .bp{color:#999}.highlight .vc,.highlight .vg,.highlight .vi{color:#008080}.highlight .il{color:#099}.highlight .gc{color:#999;background-color:#eaf2f5}.type-csharp .highlight .k,.type-csharp .highlight .kt{color:#00F}.type-csharp .highlight .nf{color:#000;font-weight:normal}.type-csharp .highlight .nc{color:#2b91af}.type-csharp .highlight .nn{color:#000}.type-csharp .highlight .s,.type-csharp .highlight .sc{color:#a31515} From 420aee5ae17e0f42b51205d6004a03cdfed1d480 Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Mon, 27 Feb 2017 15:12:04 -0500 Subject: [PATCH 047/216] Clean up build code Improve build error reporting Remove notebook relics --- build/citations.py | 7 ++-- build/references.py | 100 +++++++++++++++++++++++++------------------- 2 files changed, 60 insertions(+), 47 deletions(-) diff --git a/build/citations.py b/build/citations.py index e8b0071..9a59ba0 100644 --- a/build/citations.py +++ b/build/citations.py @@ -10,12 +10,12 @@ def validate_reference(ref): if not ref.startswith('@'): - return f'{ref} does not start with @' + return f'{ref} → does not start with @' source, tail = ref.lstrip('@').split(':', 1) if not tail: - return f'{ref} does not specify its source, e.g. doi or pmid' + return f'{ref} → does not specify its source, e.g. "@doi:" or "@pmid:"' if source not in {'doi', 'pmid', 'arxiv', 'url', 'tag'}: - return f'{ref} source is not valid' + return f'{ref} → source "{source}" is not valid' return None @@ -131,6 +131,7 @@ def citeproc_passthrough(csl_item, set_id=None): csl_item['id'] = set_id # Correct invalid CSL item types + # See https://github.com/CrossRef/rest-api-doc/issues/187 old_type = csl_item['type'] csl_item['type'] = citeproc_type_fixer.get(old_type, old_type) diff --git a/build/references.py b/build/references.py index f664589..b33e86b 100644 --- a/build/references.py +++ b/build/references.py @@ -1,15 +1,15 @@ - +#!/usr/bin/env python # coding: utf-8 - -# # Process citations and retrieve metadata - -# In[1]: +""" +Process citations and retrieve metadata +""" import json import re import os import pathlib import subprocess +import textwrap import pandas @@ -22,25 +22,46 @@ validate_reference, ) +# Run only as a script +assert __name__ == '__main__' + +def get_divider(title='Error', linewidth=79, fill_character='#'): + """ + Useful for separating sections in logs + """ + lines = [ + '', + fill_character * linewidth, + f' {title} '.center(linewidth, fill_character), + ] + return '\n'.join(lines) -# In[2]: +# Configure directories ref_dir = pathlib.Path('../references') gen_dir = ref_dir.joinpath('generated') gen_dir.mkdir(exist_ok=True) -# In[3]: - +# Read and process manuscript text = get_text('../sections') refs = sorted(get_references_from_text(text)) + +# Warn about non-failing misformatted references warn_refs = get_brackets_without_reference(text) if warn_refs: - print('WARNING: The following bracketed texts are not references:') + print(get_divider('References Warning')) + print('Potentially misformatted references detected:') print('\n'.join(warn_refs)) + +# Failing references bad_refs = list(filter(None, map(validate_reference, refs))) if bad_refs: + print(get_divider('References Error')) + print('Misformatted references detected:') print('\n'.join(bad_refs)) - assert False + raise SystemExit(1) + + ref_df = pandas.DataFrame({'text': refs}) tag_df = pandas.read_table(ref_dir.joinpath('tags.tsv')) tag_df['text'] = '@tag:' + tag_df.tag @@ -48,24 +69,19 @@ ref_df.citation.fillna(ref_df.text.str.lstrip('@'), inplace=True) -# In[4]: - -ref_df.head(3) - - -# In[5]: - def get_standard_citatation(citation, cache): + """ + For a citation, return (standard_citation, citation_metadata). + Returns (None, None) if citation metadata not retrievable. + """ try: metadata = citation_to_metadata(citation, cache) return metadata['standard_citation'], metadata['citation_id'] except Exception as e: - print(citation, e) return None, None -# In[6]: - +# Load metadata cache cache_path = gen_dir.joinpath('citations.json') use_cache = cache_path.exists() and 'REFRESH_METADATA_CACHE' not in os.environ print('Using metadata cache:', use_cache) @@ -75,51 +91,48 @@ def get_standard_citatation(citation, cache): else: metadata_cache = {} -ref_df['standard_citation'], ref_df['citation_id'] = zip(*ref_df.citation.apply( - get_standard_citatation, cache=metadata_cache)) - +# Get metadata and populate standard_citation and citation_id columns +ref_df['standard_citation'], ref_df['citation_id'] = zip( + *ref_df.citation.apply(get_standard_citatation, cache=metadata_cache)) -# In[7]: - -ref_df.head(3) - - -# In[8]: +broken_citations = ref_df[ref_df.citation_id.isnull()] +if not broken_citations.empty: + bad = '\n'.join(broken_citations['text']) + message = get_divider('References Error') + textwrap.dedent(f''' + Metadata could not be retrieved for the following citations: + {bad} + ''') + raise SystemExit(message) +# Duplicated citations print(f''' {len(ref_df)} unique citations strings extracted from text {ref_df.standard_citation.nunique()} unique citations after standardizations '''.strip()) - -# In[9]: - -# Duplicated citations dup_df = ref_df[ref_df.standard_citation.duplicated(keep=False)] print(dup_df) -# In[10]: - +# Convert to numbered refernces for pandoc converted_text = text for old, new in zip(ref_df.text, '@' + ref_df.citation_id): old = re.escape(old) converted_text = re.sub(old + '(?=[\s\]])', new, converted_text) +# Write manuscript for pandoc with gen_dir.joinpath('all-sections.md').open('wt') as write_file: write_file.write(converted_text) - -# In[11]: - +# Write citation table path = gen_dir.joinpath('processed-citations.tsv') ref_df.to_csv(path, sep='\t', index=False) +# Write metadata cache with cache_path.open('wt') as write_file: json.dump(metadata_cache, write_file, indent=2, ensure_ascii=False) -# In[12]: - +# Generate CSL items for Pandoc csl_items = list() bibtex_stanzas = list() for metadata in metadata_cache.values(): @@ -128,17 +141,16 @@ def get_standard_citatation(citation, cache): elif 'bibtex' in metadata: bibtex_stanzas.append(metadata['bibtex']) +# Write bibtex bibliography bib_path = gen_dir.joinpath('bibliography.bib') with bib_path.open('wt') as write_file: write_file.write('\n'.join(bibtex_stanzas)) +# Convert bibtex records to JSON CSL Items bib_items = subprocess.check_output(['pandoc-citeproc', '--bib2json', bib_path]) bib_items = json.loads(bib_items) csl_items.extend(map(citeproc_passthrough, bib_items)) - -# In[13]: - +# Write JSON CSL bibliography for Pandoc. with gen_dir.joinpath('bibliography.json').open('wt') as write_file: json.dump(csl_items, write_file, indent=2, ensure_ascii=False) - From efa428fe4f5758c397746d8b482ee40edc7aedfe Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Mon, 27 Feb 2017 15:37:52 -0500 Subject: [PATCH 048/216] Improve README CI and License doc --- README.md | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index cb0b379..60c3c68 100644 --- a/README.md +++ b/README.md @@ -84,10 +84,25 @@ review will cover. [![Build Status](https://travis-ci.org/greenelab/deep-review.svg?branch=master)](https://travis-ci.org/greenelab/deep-review) -[`.travis.yml`](.travis.yml) configures the continuous integration. +When you make a pull request, Travis CI will test whether your changes break the build process to generate the formatted manuscript. +The build process aims to detect common errors, such as invalid references. +If your build fails, see the Travis CI logs for the cause of failure and revise your pull request accordingly. + +When a pull request is merged, Travis CI performs the build and writes the results to the [`gh-pages`](https://github.com/greenelab/deep-review/tree/gh-pages) and [`references`](https://github.com/greenelab/deep-review/tree/references) branches. +The `gh-pages` branch hosts the following URLs: + ++ **HTML manuscript** at https://greenelab.github.io/deep-review/
+ short URL: https://git.io/vytJN ++ **PDF manuscript** at https://greenelab.github.io/deep-review/deep-review.pdf
+ short URL: https://git.io/vytJ5 + +For continuous integration configuration details, see [`.travis.yml`](.travis.yml). ## License +[![License: CC BY 4.0](https://img.shields.io/badge/License%20All-CC%20BY%204.0-lightgrey.svg)](http://creativecommons.org/licenses/by/4.0/) +[![License: CC0 1.0](https://img.shields.io/badge/License%20Parts-CC0%201.0-lightgrey.svg)](https://creativecommons.org/publicdomain/zero/1.0/) + This entirety of this repository is licensed under a CC BY 4.0 License ([`LICENSE.md`](LICENSE.md)), which allows reuse with attribution. Please attribute by linking to https://github.com/greenelab/deep-review. From b0f1a306afcdf3c81d2ca8b1ffa674834177e403 Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Fri, 3 Mar 2017 10:59:12 -0500 Subject: [PATCH 049/216] Pandoc: link-citations to bibliography (#272) See #271 --- build/build.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/build/build.sh b/build/build.sh index 12745c8..245fdba 100644 --- a/build/build.sh +++ b/build/build.sh @@ -19,6 +19,7 @@ pandoc --verbose \ --from=markdown --to=html \ --bibliography=$BIBLIOGRAPHY_PATH \ --csl=$CSL_PATH \ + --metadata link-citations=true \ --css=github-pandoc.css \ --output=output/index.html \ $INPUT_PATH @@ -30,5 +31,6 @@ pandoc \ --to=html5 \ --bibliography=$BIBLIOGRAPHY_PATH \ --csl=$CSL_PATH \ + --metadata link-citations=true \ --output=output/deep-review.pdf \ $INPUT_PATH From bc7285593ff0ccf9ffe37e62157a32348c631e8b Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Fri, 3 Mar 2017 12:01:48 -0500 Subject: [PATCH 050/216] Switch Travis to trusty from precise (#273) Improve build wait times. See https://docs.travis-ci.com/user/trusty-ci-environment/ --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index eca4d51..ba6e2a5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,10 @@ +dist: trusty +sudo: false language: generic branches: only: - master before_install: - - sudo apt-get update --quiet --quiet - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh --output-document miniconda.sh - bash miniconda.sh -b -p $HOME/miniconda From 3b763ff72b4607c95ad3df05b3a1cb366fa901c8 Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Sat, 4 Mar 2017 18:20:42 -0500 Subject: [PATCH 051/216] Timestamp manuscripts with OpenTimestamps (#274) Closes https://github.com/greenelab/deep-review/issues/269 --- .gitignore | 1 + .gitmodules | 3 +++ build/environment.yml | 1 + ci/deploy.sh | 10 ++++++++++ ci/opentimestamps-client | 1 + output/README.md | 18 ++++++++++++++++++ 6 files changed, 34 insertions(+) create mode 100644 .gitmodules create mode 160000 ci/opentimestamps-client create mode 100644 output/README.md diff --git a/.gitignore b/.gitignore index 0c962c3..64e8cee 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ # Generated manuscript output files output/index.html output/deep-review.pdf +output/*.ots # Generated reference files references/generated/* diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..692574b --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "ci/opentimestamps-client"] + path = ci/opentimestamps-client + url = https://github.com/opentimestamps/opentimestamps-client.git diff --git a/build/environment.yml b/build/environment.yml index ff69c11..f834e36 100644 --- a/build/environment.yml +++ b/build/environment.yml @@ -12,3 +12,4 @@ dependencies: - arxiv2bib==1.0.7 - bibtexparser==0.6.2 - ghp-import==0.5.5 + - python-bitcoinlib==0.7.0 diff --git a/ci/deploy.sh b/ci/deploy.sh index e812c34..26a9a44 100644 --- a/ci/deploy.sh +++ b/ci/deploy.sh @@ -1,5 +1,15 @@ +# Exit on errors set -o errexit +# Add commit hash to the README +envsubst < output/README.md > output/README.md + +# Generate OpenTimestamps +python ci/opentimestamps-client/ots stamp \ + output/index.html \ + output/deep-review.pdf \ + output/README.md + # Configure git git config --global push.default simple git config --global user.email `git log --max-count=1 --format='%ae'` diff --git a/ci/opentimestamps-client b/ci/opentimestamps-client new file mode 160000 index 0000000..9fc5528 --- /dev/null +++ b/ci/opentimestamps-client @@ -0,0 +1 @@ +Subproject commit 9fc55286be24b886180b337609d0c389ebc3f257 diff --git a/output/README.md b/output/README.md new file mode 100644 index 0000000..5e2df76 --- /dev/null +++ b/output/README.md @@ -0,0 +1,18 @@ +# Output directory containing the formatted manuscript + +The [`gh-pages`](https://github.com/greenelab/deep-review/tree/gh-pages) branch hosts the contents of this directory at https://greenelab.github.io/deep-review/. + +## Files + +This directory contains the following files, which are mostly ignored on the `master` branch: + ++ [`index.html`](index.html) is an HTML manuscript. ++ [`github-pandoc.css`](github-pandoc.css) sets the display style for `index.html`. ++ [`deep-review.pdf`](deep-review.pdf) is a PDF manuscript. ++ `*.ots` files are OpenTimestamps which can be used to verify manuscript existence at or before a given time. + [OpenTimestamps](opentimestamps.org) uses the Bitcoin blockchain to attest to file hash existence. + +## Source + +The manuscripts in this directory were built from +[`$TRAVIS_COMMIT`](https://github.com/greenelab/deep-review/commit/$TRAVIS_COMMIT). From 2d589e4839be73e08ff8a8870b731a729fc671ca Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Sat, 4 Mar 2017 18:56:11 -0500 Subject: [PATCH 052/216] Fix blank output/README.md bug (#275) Affects gh-pages branch only. Fixed to allow reading from and writing to the same file. See http://stackoverflow.com/a/3897483/4651668 Bug introduced in https://github.com/greenelab/deep-review/pull/274. --- ci/deploy.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/deploy.sh b/ci/deploy.sh index 26a9a44..4c1b936 100644 --- a/ci/deploy.sh +++ b/ci/deploy.sh @@ -2,7 +2,8 @@ set -o errexit # Add commit hash to the README -envsubst < output/README.md > output/README.md +envsubst < output/README.md > output/README-complete.md +mv output/README-complete.md output/README.md # Generate OpenTimestamps python ci/opentimestamps-client/ots stamp \ From c02ef7d361b2ccc35e6d8d366045b6d1164be1d1 Mon Sep 17 00:00:00 2001 From: Casey Greene Date: Fri, 17 Mar 2017 10:03:46 -0400 Subject: [PATCH 053/216] update expected date (#279) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 60c3c68..ce5df37 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ We have the opportunity to write a headline review for *Journal of the Royal Society Interface* on a topic overlapping the computer and life sciences in the area of systems pharmacology. Our aim is to have the paper ready to submit by -*January 15, 2017*. +*April 15, 2017*. > A Headline Review is one in a short, targeted series of high-level reviews within a particular topic of a burgeoning research area. We encourage authors to From c0c8e835c2df153cb8f7bbab354404578a23f5e6 Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Fri, 7 Apr 2017 14:14:17 -0400 Subject: [PATCH 054/216] Style: flake8 formatting in references.py --- build/references.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/build/references.py b/build/references.py index b33e86b..62ce529 100644 --- a/build/references.py +++ b/build/references.py @@ -25,6 +25,7 @@ # Run only as a script assert __name__ == '__main__' + def get_divider(title='Error', linewidth=79, fill_character='#'): """ Useful for separating sections in logs @@ -115,8 +116,9 @@ def get_standard_citatation(citation, cache): # Convert to numbered refernces for pandoc converted_text = text -for old, new in zip(ref_df.text, '@' + ref_df.citation_id): +for old, new in zip(ref_df.text, ref_df.citation_id): old = re.escape(old) + new = f'@{new}' converted_text = re.sub(old + '(?=[\s\]])', new, converted_text) # Write manuscript for pandoc @@ -147,7 +149,8 @@ def get_standard_citatation(citation, cache): write_file.write('\n'.join(bibtex_stanzas)) # Convert bibtex records to JSON CSL Items -bib_items = subprocess.check_output(['pandoc-citeproc', '--bib2json', bib_path]) +bib_items = subprocess.check_output( + ['pandoc-citeproc', '--bib2json', bib_path]) bib_items = json.loads(bib_items) csl_items.extend(map(citeproc_passthrough, bib_items)) From d3ee4760eefb2b9e966990e43f6122db55af58e8 Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Fri, 7 Apr 2017 14:19:34 -0400 Subject: [PATCH 055/216] Style: flake8 citations.py --- build/citations.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/build/citations.py b/build/citations.py index 9a59ba0..6113a8b 100644 --- a/build/citations.py +++ b/build/citations.py @@ -2,7 +2,6 @@ import pathlib import re -import pandas import bibtexparser import metadata @@ -93,14 +92,16 @@ def citation_to_metadata(citation, cache={}): else: msg = f'Unsupported citation source {source} in {citation}' raise ValueError(msg) - + citation_id = f'ref_{len(cache)}' result['citation_id'] = citation_id if 'citeproc' in result: - result['citeproc'] = citeproc_passthrough(result['citeproc'], set_id=citation_id) + result['citeproc'] = citeproc_passthrough( + result['citeproc'], set_id=citation_id) if 'bibtex' in result: - result['bibtex'] = bibtex_passthrough(result['bibtex'], set_id=citation_id) - + result['bibtex'] = bibtex_passthrough( + result['bibtex'], set_id=citation_id) + cache[standard_citation] = result return result @@ -112,15 +113,17 @@ def citation_to_metadata(citation, cache={}): 'proceedings-article': 'paper-conference', } +# Remove citeproc keys to fix pandoc-citeproc errors citeproc_remove_keys = [ - # pandoc-citeproc: Error in $[0].ISSN[0]: failed to parse field ISSN: mempty + # Error in $[0].ISSN[0]: failed to parse field ISSN: mempty 'ISSN', - # pandoc-citeproc: Error in $[2].ISBN[0]: failed to parse field ISBN: mempty + # Error in $[2].ISBN[0]: failed to parse field ISBN: mempty 'ISBN', # pandoc-citeproc expected Object not array for archive 'archive', ] + def citeproc_passthrough(csl_item, set_id=None): """ Fix errors in a CSL item and optional change its id. From de283bea0fd6e7bb587d46db5afb9c3ddf65d8be Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Fri, 7 Apr 2017 14:32:27 -0400 Subject: [PATCH 056/216] pandoc-citeproc: fix multiple references in brackets Closes https://github.com/greenelab/deep-review/issues/290 --- build/citations.py | 16 +++++++++++++++- build/references.py | 3 +++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/build/citations.py b/build/citations.py index 6113a8b..ee142e4 100644 --- a/build/citations.py +++ b/build/citations.py @@ -18,12 +18,15 @@ def validate_reference(ref): return None +bracketed_reference_pattern = re.compile(r'\[(@.+?)\]', flags=re.DOTALL) + + def get_references_from_text(text): """ Extract the set of references in a text """ refs = set() - for ref_text in re.findall(r'\[(@.+?)\]', text, flags=re.DOTALL): + for ref_text in bracketed_reference_pattern.findall(text): for ref in ref_text.split(): if not ref: continue @@ -31,6 +34,17 @@ def get_references_from_text(text): return refs +def semicolon_separate_references(text): + """ + Separate multiple references inside the same brackets with a space and + semicolon for pandoc-citeproc. + """ + return bracketed_reference_pattern.sub( + repl=lambda x: '; '.join(x.group().split()), + string=text + ) + + def get_brackets_without_reference(text): """ Find bracketed text that does not start with @. Does not match diff --git a/build/references.py b/build/references.py index 62ce529..6814820 100644 --- a/build/references.py +++ b/build/references.py @@ -19,6 +19,7 @@ get_brackets_without_reference, get_references_from_text, get_text, + semicolon_separate_references, validate_reference, ) @@ -120,6 +121,8 @@ def get_standard_citatation(citation, cache): old = re.escape(old) new = f'@{new}' converted_text = re.sub(old + '(?=[\s\]])', new, converted_text) +# Semicolon separate multiple refernces for pandoc-citeproc +converted_text = semicolon_separate_references(converted_text) # Write manuscript for pandoc with gen_dir.joinpath('all-sections.md').open('wt') as write_file: From 0eaa63853accd73603702af4cfcaf60e92692f29 Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Mon, 10 Apr 2017 16:00:39 -0400 Subject: [PATCH 057/216] Replace numbered citation_id with 5-byte blake2 hash (#298) * Replace numbered citation_id with 5-byte blake2 hash * Use base32 encoding for 5 bytes in 8 chars * Check for citation_id collisions --- build/citations.py | 5 ++++- build/references.py | 8 ++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/build/citations.py b/build/citations.py index ee142e4..f8e8130 100644 --- a/build/citations.py +++ b/build/citations.py @@ -1,4 +1,6 @@ +import base64 import collections +from hashlib import blake2b import pathlib import re @@ -107,7 +109,8 @@ def citation_to_metadata(citation, cache={}): msg = f'Unsupported citation source {source} in {citation}' raise ValueError(msg) - citation_id = f'ref_{len(cache)}' + digest = blake2b(standard_citation.encode(), digest_size=5).digest() + citation_id = base64.b32encode(digest).decode() result['citation_id'] = citation_id if 'citeproc' in result: result['citeproc'] = citeproc_passthrough( diff --git a/build/references.py b/build/references.py index 6814820..5c72883 100644 --- a/build/references.py +++ b/build/references.py @@ -106,6 +106,14 @@ def get_standard_citatation(citation, cache): ''') raise SystemExit(message) +# Check that no two standard_citations have the same citation_id +# which could occur due to a hash collision +collision_df = ref_df[['standard_citation', 'citation_id']].drop_duplicates() +collision_df = collision_df[collision_df.citation_id.duplicated(keep=False)] +if not collision_df.empty: + print(collision_df) + raise SystemExit(f'OMF! Hash collision. Congratulations.') + # Duplicated citations print(f''' {len(ref_df)} unique citations strings extracted from text From 72432d736c32805ac2b88c6cbee2cd7457d6a626 Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Mon, 10 Apr 2017 16:44:15 -0400 Subject: [PATCH 058/216] Fix indentation bug (#303) --- build/references.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build/references.py b/build/references.py index 5c72883..0482328 100644 --- a/build/references.py +++ b/build/references.py @@ -100,10 +100,10 @@ def get_standard_citatation(citation, cache): broken_citations = ref_df[ref_df.citation_id.isnull()] if not broken_citations.empty: bad = '\n'.join(broken_citations['text']) - message = get_divider('References Error') + textwrap.dedent(f''' + message = get_divider('References Error') + textwrap.dedent(''' Metadata could not be retrieved for the following citations: - {bad} - ''') + {} + ''').format(bad) raise SystemExit(message) # Check that no two standard_citations have the same citation_id From 76b8ab8bd36f26b4eb2b5bcde6a91778a814c589 Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Tue, 11 Apr 2017 12:26:54 -0400 Subject: [PATCH 059/216] Encode citation_ids using base62 --- build/citations.py | 6 +++--- build/environment.yml | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/build/citations.py b/build/citations.py index f8e8130..87e2816 100644 --- a/build/citations.py +++ b/build/citations.py @@ -1,9 +1,9 @@ -import base64 import collections from hashlib import blake2b import pathlib import re +import base62 import bibtexparser import metadata @@ -109,8 +109,8 @@ def citation_to_metadata(citation, cache={}): msg = f'Unsupported citation source {source} in {citation}' raise ValueError(msg) - digest = blake2b(standard_citation.encode(), digest_size=5).digest() - citation_id = base64.b32encode(digest).decode() + digest = blake2b(standard_citation.encode(), digest_size=6).digest() + citation_id = base62.encodebytes(digest) result['citation_id'] = citation_id if 'citeproc' in result: result['citeproc'] = citeproc_passthrough( diff --git a/build/environment.yml b/build/environment.yml index f834e36..21a6936 100644 --- a/build/environment.yml +++ b/build/environment.yml @@ -10,6 +10,7 @@ dependencies: - requests=2.13.0 - pip: - arxiv2bib==1.0.7 + - pybase62==0.2.0 - bibtexparser==0.6.2 - ghp-import==0.5.5 - python-bitcoinlib==0.7.0 From e11f8ea88c78f61859233ddf0a47d3a507d06baa Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Thu, 13 Apr 2017 08:25:18 -0400 Subject: [PATCH 060/216] CSL: remove malformated event field The event field of CSL data should be a string (https://git.io/vSSug). Instead the event key returns an object as shown below: ```sh curl --location \ --header "Accept: application/vnd.citationstyles.csl+json" \ https://doi.org/10.1109/icnn.1994.374637 \ | python -m json.tool ``` ```json "event": { "name": "1994 IEEE International Conference on Neural Networks (ICNN'94)", "location": "Orlando, FL, USA", "acronym": "ICNN-94" }, ``` Related to https://github.com/CrossRef/rest-api-doc/issues/187. DOI content negotiation returns invalid CSL --- build/citations.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/build/citations.py b/build/citations.py index 87e2816..c84fece 100644 --- a/build/citations.py +++ b/build/citations.py @@ -138,6 +138,8 @@ def citation_to_metadata(citation, cache={}): 'ISBN', # pandoc-citeproc expected Object not array for archive 'archive', + # failed to parse field event: Could not read as string + 'event', ] From dedd0217a8dc14e31c8b2f9a146b62ab15ed813c Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Thu, 13 Apr 2017 09:08:09 -0400 Subject: [PATCH 061/216] Remove reference lists in CSL bibliography.json (#309) Assuming the `reference` field was recently added. See https://doi.org/10.1126/science.aal1012. --- build/citations.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/build/citations.py b/build/citations.py index c84fece..597218c 100644 --- a/build/citations.py +++ b/build/citations.py @@ -140,6 +140,8 @@ def citation_to_metadata(citation, cache={}): 'archive', # failed to parse field event: Could not read as string 'event', + # remove the references of cited papers. Not neccessary and unwieldy. + 'reference', ] From 356668e97999e61e0c4b4c7634a4935162b6554d Mon Sep 17 00:00:00 2001 From: Casey Greene Date: Thu, 13 Apr 2017 09:58:14 -0400 Subject: [PATCH 062/216] add timeline; notifications dates; etc (#310) --- README.md | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ce5df37..e267eb7 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,17 @@ We have the opportunity to write a headline review for *Journal of the Royal Society Interface* on a topic overlapping the computer and life sciences in the -area of systems pharmacology. Our aim is to have the paper ready to submit by -*April 15, 2017*. +area of systems pharmacology. We are hitting the home stretch towards getting +this paper submitted. Consequently, we have established the following +guidelines: + +* April 24: Any empty section, even if in the outline, gets dropped due to lack + of interest. +* May 1: Any section not merged & ready for revisions gets dropped due to lack + of completion. Invitations go out to contributing authors to approve the + manuscript. +* May 1 - 14: Revisions, flow, clarity, and author approval. Authors must approve + by May 14 to be included as co-authors. > A Headline Review is one in a short, targeted series of high-level reviews within a particular topic of a burgeoning research area. We encourage authors to From 432232d1d760e58685e47628eb8c78c628d0163d Mon Sep 17 00:00:00 2001 From: Casey Greene Date: Wed, 19 Apr 2017 12:34:34 -0400 Subject: [PATCH 063/216] provide more info for contributors (#318) --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index e267eb7..06b2eef 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,13 @@ guidelines: manuscript. * May 1 - 14: Revisions, flow, clarity, and author approval. Authors must approve by May 14 to be included as co-authors. + +The most current version of the `master` branch is built by continuous integration +and [available via gh-pages](https://greenelab.github.io/deep-review/). To see +what's incoming, check the +[open pull requests](https://github.com/greenelab/deep-review/pulls). +If you want to claim a section (time is short - see timeline above!) issue number +[188](https://github.com/greenelab/deep-review/issues/188) is the place to do it. > A Headline Review is one in a short, targeted series of high-level reviews within a particular topic of a burgeoning research area. We encourage authors to From 0f5a32250dc6223b303d0716235a8d559bdaed8b Mon Sep 17 00:00:00 2001 From: Paul Agapow Date: Mon, 24 Apr 2017 13:51:51 +0100 Subject: [PATCH 064/216] expanded section on standards, inc. refs to ongoing projects --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 64e8cee..0f1030f 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,7 @@ __pycache__/ # Jupyter Notebook .ipynb_checkpoints + +# Misc temporary files +*.bak + From 6aa02275dcb658e200bc9507575338f332ca265c Mon Sep 17 00:00:00 2001 From: Casey Greene Date: Tue, 9 May 2017 12:12:42 -0400 Subject: [PATCH 065/216] Author approval (#389) * add initial approval/contact form * add grants to cite col * update readme to match --- README.md | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 06b2eef..cf024ce 100644 --- a/README.md +++ b/README.md @@ -13,9 +13,15 @@ guidelines: * May 1: Any section not merged & ready for revisions gets dropped due to lack of completion. Invitations go out to contributing authors to approve the manuscript. -* May 1 - 14: Revisions, flow, clarity, and author approval. Authors must approve - by May 14 to be included as co-authors. - +* May 1 - 14: Revisions, flow, and clarity. +* Final edits and author approval. Authors must approve by [Noon Eastern + Time](https://time.is/ET) on May 17 to be included as co-authors. File a pull + request to add the appropriate line to `author_contact_approval.md` + +At this stage it is ideal if **each pull request touches no more than one +subsection** as this minimizes the number of people who will need to sign off on +changes. + The most current version of the `master` branch is built by continuous integration and [available via gh-pages](https://greenelab.github.io/deep-review/). To see what's incoming, check the @@ -111,7 +117,7 @@ The `gh-pages` branch hosts the following URLs: short URL: https://git.io/vytJN + **PDF manuscript** at https://greenelab.github.io/deep-review/deep-review.pdf
short URL: https://git.io/vytJ5 - + For continuous integration configuration details, see [`.travis.yml`](.travis.yml). ## License From bd99c5c2e04980a07983cd35fc713e9f6fdfacd0 Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Wed, 17 May 2017 09:17:32 -0400 Subject: [PATCH 066/216] Bibtex: use url not link attribute (#465) --- build/citations.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build/citations.py b/build/citations.py index 597218c..0759041 100644 --- a/build/citations.py +++ b/build/citations.py @@ -185,6 +185,9 @@ def bibtex_passthrough(text, set_id=None): entry, = bibdb.entries if 'author' in entry: entry['author'] = ' and '.join(entry['author'].rstrip(';').split('; ')) + # Set URL as url rather than link attribute + if 'link' in entry and 'url' not in entry: + entry['url'] = entry.pop('link') if set_id is not None: entry['ID'] = set_id return bibtexparser.dumps(bibdb) From 5e611e64de1a4188342a8704243df29d47a3da68 Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Wed, 17 May 2017 09:18:34 -0400 Subject: [PATCH 067/216] Get metadata for URL references using Greycite (#464) Refs https://github.com/greenelab/deep-review/issues/381 * Enable Greycite URL metadata fetching * Use urllib not requests for Greycite API Refs https://github.com/kennethreitz/requests/issues/4023 * Fix URL references for Greycite http://www.ncrr.nih.gov/publications/informatics/ehr.pdf is a broken link for a 2006 report titled "Electronic Health Records Overview" by the MITRE Corporation. See https://www.scribd.com/document/51496338 Greycite was struggling with http://dl.acm.org/citation.cfm?id=2188395 Returned a 404 for its API call. Replaced with an alternative location. * Greycite workarounds --- build/metadata.py | 60 ++++++++++++++++++++++++++++++++++++----------- 1 file changed, 46 insertions(+), 14 deletions(-) diff --git a/build/metadata.py b/build/metadata.py index cd3c1eb..354edcd 100644 --- a/build/metadata.py +++ b/build/metadata.py @@ -1,3 +1,6 @@ +import json +import re +import urllib.parse import urllib.request import requests @@ -53,29 +56,58 @@ def get_pubmed_citeproc(pubmed_id): return response.json() -# def get_url_citeproc(url): -# """ -# Uses greycite (unreliable) -# https://github.com/cboettig/knitcitations/blob/master/R/greycite.R -# """ -# params = {'uri': url} -# url = 'http://greycite.knowledgeblog.org/' -# response = requests.get(url, params) -# print(response.url) -# return response +def get_url_citeproc_greycite(url): + """ + Uses Greycite which has experiened uptime problems in the past. + API calls seem to take at least 15 seconds. Browser requests are much + faster. Setting header did not have an effect. Consider mimicking browser + using selenium. + + More information on Greycite at: + http://greycite.knowledgeblog.org/ + http://knowledgeblog.org/greycite + https://arxiv.org/abs/1304.7151 + https://git.io/v9N2C + + Uses urllib.request.urlopen rather than requests.get due to + https://github.com/kennethreitz/requests/issues/4023 + """ + encoded = urllib.parse.quote_plus(url) + api_url = f'http://greycite.knowledgeblog.org/json?uri={encoded}' + response = urllib.request.urlopen(api_url) + content = response.read() + # Some Greycite responses were valid JSON besides for an error appended + # like "

*** Date set from uri

" or "

*** fetch error : 404

". + pattern = re.compile(b"

\*\*\*.*

") + content = pattern.sub(b'', content) + csl_item = json.loads(content) + csl_item['type'] = 'webpage' + return csl_item -def get_url_citeproc(url): + +def get_url_citeproc_manual(url): """ - Uses greycite (unreliable) - https://github.com/cboettig/knitcitations/blob/master/R/greycite.R + Manually create citeproc for a URL. """ return { 'URL': url, - 'type': 'webpage' + 'type': 'webpage', } +def get_url_citeproc(url): + """ + Get citeproc for a URL trying a sequence of strategies. + """ + try: + return get_url_citeproc_greycite(url) + except Exception as e: + print(f'Error getting {url} from Greycite: {e}') + # Fallback strategy + return get_url_citeproc_manual(url) + + def get_doi_bibtex(doi): """ Use DOI Content Negotioation (http://crosscite.org/cn/) to retrieve a From 7762bfecc6c0a6ce2ae1113d252fd5b766bf7e61 Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Wed, 17 May 2017 14:06:22 -0400 Subject: [PATCH 068/216] Use requests for Greycite (#472) Uses `Connection: close` workaround by @Lukasa. See https://github.com/kennethreitz/requests/issues/4023#issuecomment-302164405 --- build/metadata.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/build/metadata.py b/build/metadata.py index 354edcd..279f569 100644 --- a/build/metadata.py +++ b/build/metadata.py @@ -1,6 +1,5 @@ import json import re -import urllib.parse import urllib.request import requests @@ -72,16 +71,16 @@ def get_url_citeproc_greycite(url): Uses urllib.request.urlopen rather than requests.get due to https://github.com/kennethreitz/requests/issues/4023 """ - encoded = urllib.parse.quote_plus(url) - api_url = f'http://greycite.knowledgeblog.org/json?uri={encoded}' - response = urllib.request.urlopen(api_url) - content = response.read() - + response = requests.get( + 'http://greycite.knowledgeblog.org/json', + params={'uri': url}, + headers={'Connection': 'close'}, + ) # Some Greycite responses were valid JSON besides for an error appended # like "

*** Date set from uri

" or "

*** fetch error : 404

". - pattern = re.compile(b"

\*\*\*.*

") - content = pattern.sub(b'', content) - csl_item = json.loads(content) + pattern = re.compile(r"

\*\*\*.*

") + text = pattern.sub('', response.text) + csl_item = json.loads(text) csl_item['type'] = 'webpage' return csl_item From cdd819a26abdc190e8ea8b60991147b975f26827 Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Thu, 18 May 2017 12:50:01 -0400 Subject: [PATCH 069/216] Count references by type (#479) --- build/references.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/build/references.py b/build/references.py index 0482328..708757b 100644 --- a/build/references.py +++ b/build/references.py @@ -123,7 +123,18 @@ def get_standard_citatation(citation, cache): dup_df = ref_df[ref_df.standard_citation.duplicated(keep=False)] print(dup_df) -# Convert to numbered refernces for pandoc +# Number of distinct references by type +type_counts = ( + ref_df + .standard_citation + .drop_duplicates() + .map(lambda x: x.split(':')[0]) + .value_counts() +) +print('References by type:') +print(type_counts) + +# Convert to citation_id citations for pandoc converted_text = text for old, new in zip(ref_df.text, ref_df.citation_id): old = re.escape(old) From ca09d920dea6644ec514e6340c816a7ef2a416c2 Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Thu, 18 May 2017 12:58:50 -0400 Subject: [PATCH 070/216] Fix GitHub markdown numbered list rendering (#481) --- CONTRIBUTING.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e723de3..f93b563 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -48,10 +48,10 @@ The custom citation guidelines are as follows: 1. Always use a DOI for the version of record if available. Cite DOIs like `[@doi:10.15363/thinklab.4]`. -+ If the version of record doesn't have a DOI but does have a PubMed ID, cite +2. If the version of record doesn't have a DOI but does have a PubMed ID, cite like `[@pmid:26158728]`. -+ If the article is an _arXiv_ preprint, cite like `[@arxiv:1508.06576]`. -+ If and only if the article has none of the above, cite with the URL like +3. If the article is an _arXiv_ preprint, cite like `[@arxiv:1508.06576]`. +4. If and only if the article has none of the above, cite with the URL like `[@url:http://openreview.net/pdf?id=Sk-oDY9ge]`. You cite multiple items at once like `[@doi:10.15363/thinklab.4 @pmid:26158728 From 7c220c5bee3f08d853adfda25ff2ccb789115fac Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Thu, 18 May 2017 13:44:17 -0400 Subject: [PATCH 071/216] Upgrade opentimestamps-client (#482) * Upgrade opentimestamps-client Upgrades to https://github.com/opentimestamps/opentimestamps-client/commit/e1ff9833b3b3acecf49c4b2c682186df04902ac3 * Add pysha3 PyPI dependency --- build/environment.yml | 3 ++- ci/opentimestamps-client | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/build/environment.yml b/build/environment.yml index 21a6936..90b66fe 100644 --- a/build/environment.yml +++ b/build/environment.yml @@ -10,7 +10,8 @@ dependencies: - requests=2.13.0 - pip: - arxiv2bib==1.0.7 - - pybase62==0.2.0 - bibtexparser==0.6.2 - ghp-import==0.5.5 + - pybase62==0.2.0 + - pysha3==1.0.2 - python-bitcoinlib==0.7.0 diff --git a/ci/opentimestamps-client b/ci/opentimestamps-client index 9fc5528..e1ff983 160000 --- a/ci/opentimestamps-client +++ b/ci/opentimestamps-client @@ -1 +1 @@ -Subproject commit 9fc55286be24b886180b337609d0c389ebc3f257 +Subproject commit e1ff9833b3b3acecf49c4b2c682186df04902ac3 From d5f29a0de8849b3ede0ed47922b0739850f396d2 Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Fri, 19 May 2017 13:36:14 -0400 Subject: [PATCH 072/216] Enable templating with jinja2 (#485) --- build/environment.yml | 1 + build/references.py | 23 +++++++++++++++++++---- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/build/environment.yml b/build/environment.yml index 90b66fe..a98ccee 100644 --- a/build/environment.yml +++ b/build/environment.yml @@ -5,6 +5,7 @@ channels: dependencies: - bioconda::wkhtmltopdf=0.12.3 - conda-forge::pandoc=1.19.2 + - jinja2=2.9.6 - pandas=0.19.2 - python=3.6.0 - requests=2.13.0 diff --git a/build/references.py b/build/references.py index 708757b..780b416 100644 --- a/build/references.py +++ b/build/references.py @@ -4,13 +4,16 @@ Process citations and retrieve metadata """ +import collections +import json import json -import re import os import pathlib +import re import subprocess import textwrap +import jinja2 import pandas from citations import ( @@ -39,6 +42,9 @@ def get_divider(title='Error', linewidth=79, fill_character='#'): return '\n'.join(lines) +# Manuscript statistics +stats = collections.OrderedDict() + # Configure directories ref_dir = pathlib.Path('../references') gen_dir = ref_dir.joinpath('generated') @@ -124,15 +130,20 @@ def get_standard_citatation(citation, cache): print(dup_df) # Number of distinct references by type -type_counts = ( +ref_counts = ( ref_df .standard_citation .drop_duplicates() .map(lambda x: x.split(':')[0]) - .value_counts() + .pipe(collections.Counter) ) +ref_counts['total'] = sum(ref_counts.values()) +stats['reference_counts'] = ref_counts print('References by type:') -print(type_counts) +print(ref_counts) + +with gen_dir.joinpath('stats.json').open('wt') as write_file: + json.dump(stats, write_file, indent=2) # Convert to citation_id citations for pandoc converted_text = text @@ -143,6 +154,10 @@ def get_standard_citatation(citation, cache): # Semicolon separate multiple refernces for pandoc-citeproc converted_text = semicolon_separate_references(converted_text) +# Template using jina2 +template = jinja2.Template(converted_text) +converted_text = template.render(**stats) + # Write manuscript for pandoc with gen_dir.joinpath('all-sections.md').open('wt') as write_file: write_file.write(converted_text) From 1efe95423018cdbeef3afa43ce95a6a058b92498 Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Fri, 19 May 2017 13:56:35 -0400 Subject: [PATCH 073/216] Add continuous collaborative manuscript drafting section (#477) * Section: Continuous collaborative manuscript drafting * Add reference by type counts https://travis-ci.org/greenelab/deep-review/builds/233712348#L396-L400 * Revisions based on review comments * Template reference counts Refs https://github.com/greenelab/deep-review/pull/485 * Rearrange sections * Remove duplicated import --- build/references.py | 1 - 1 file changed, 1 deletion(-) diff --git a/build/references.py b/build/references.py index 780b416..5776eec 100644 --- a/build/references.py +++ b/build/references.py @@ -6,7 +6,6 @@ import collections import json -import json import os import pathlib import re From 4eeb40476e2b246ea394c60ab89a3e12c64f9620 Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Fri, 19 May 2017 16:16:31 -0400 Subject: [PATCH 074/216] Improve duplicate citation text output (#488) --- build/references.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/build/references.py b/build/references.py index 5776eec..cff33c0 100644 --- a/build/references.py +++ b/build/references.py @@ -125,8 +125,11 @@ def get_standard_citatation(citation, cache): {ref_df.standard_citation.nunique()} unique citations after standardizations '''.strip()) -dup_df = ref_df[ref_df.standard_citation.duplicated(keep=False)] -print(dup_df) +print( + ref_df[ref_df.standard_citation.duplicated(keep=False)] + .sort_values('standard_citation') + .to_string(index=False, columns=['standard_citation', 'text']) +) # Number of distinct references by type ref_counts = ( From 883fca8ab10a46ef547ac65b1c64df0b78e51568 Mon Sep 17 00:00:00 2001 From: Casey Greene Date: Tue, 23 May 2017 12:43:58 -0400 Subject: [PATCH 075/216] update title from @bdo311 closes #496 (#503) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index cf024ce..fa7819c 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Deep Learning, Genomics, and Precision Medicine +# Opportunities and obstacles for deep learning in biology and medicine [![Code Climate](https://codeclimate.com/github/greenelab/deep-review/badges/gpa.svg)](https://codeclimate.com/github/greenelab/deep-review) From 055716a48ac3f8272cff5565b3f8bbc6b2a1eedc Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Wed, 24 May 2017 12:04:49 -0400 Subject: [PATCH 076/216] Enable manual citeproc JSON for references Refs https://github.com/greenelab/deep-review/issues/381#issuecomment-303281944 In manual-references.json use standard_citation not citation_id Closes https://github.com/greenelab/deep-review/pull/506 --- build/citations.py | 8 +++++--- build/references.py | 14 ++++++++++---- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/build/citations.py b/build/citations.py index 0759041..fda8061 100644 --- a/build/citations.py +++ b/build/citations.py @@ -80,14 +80,14 @@ def standardize_identifier(source, identifier): return identifier -def citation_to_metadata(citation, cache={}): +def citation_to_metadata(citation, cache={}, override={}): """ Return a dictionary with citation metadata """ source, identifer = citation.split(':', 1) identifer = standardize_identifier(source, identifer) standard_citation = f'{source}:{identifer}' - if standard_citation in cache: + if standard_citation not in override and standard_citation in cache: return cache[standard_citation] result = { @@ -96,7 +96,9 @@ def citation_to_metadata(citation, cache={}): 'standard_citation': standard_citation } - if source == 'doi': + if standard_citation in override: + result['citeproc'] = override[standard_citation] + elif source == 'doi': result['citeproc'] = metadata.get_doi_citeproc(identifer) # result['bibtex'] = metadata.get_doi_bibtex(identifer) elif source == 'pmid': diff --git a/build/references.py b/build/references.py index cff33c0..d9e8f2b 100644 --- a/build/references.py +++ b/build/references.py @@ -76,13 +76,13 @@ def get_divider(title='Error', linewidth=79, fill_character='#'): ref_df.citation.fillna(ref_df.text.str.lstrip('@'), inplace=True) -def get_standard_citatation(citation, cache): +def get_standard_citatation(citation, cache, override): """ For a citation, return (standard_citation, citation_metadata). Returns (None, None) if citation metadata not retrievable. """ try: - metadata = citation_to_metadata(citation, cache) + metadata = citation_to_metadata(citation, cache, override) return metadata['standard_citation'], metadata['citation_id'] except Exception as e: return None, None @@ -98,9 +98,15 @@ def get_standard_citatation(citation, cache): else: metadata_cache = {} +# Read manual citations (overrides) +with ref_dir.joinpath('manual-references.json').open() as read_file: + overrides = json.load(read_file) +overrides = {ref.pop('standard_citation'): ref for ref in overrides} + # Get metadata and populate standard_citation and citation_id columns -ref_df['standard_citation'], ref_df['citation_id'] = zip( - *ref_df.citation.apply(get_standard_citatation, cache=metadata_cache)) +result = ref_df.citation.apply( + get_standard_citatation, cache=metadata_cache, override=overrides) +ref_df['standard_citation'], ref_df['citation_id'] = zip(*result) broken_citations = ref_df[ref_df.citation_id.isnull()] if not broken_citations.empty: From 1023a0ff014f118b66a17f907851a7facfb66bca Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Wed, 24 May 2017 14:02:07 -0400 Subject: [PATCH 077/216] Style: display hyperlinks for all references --- build/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/build.sh b/build/build.sh index 245fdba..04e5eaf 100644 --- a/build/build.sh +++ b/build/build.sh @@ -5,7 +5,7 @@ echo "Retrieving and processing reference metadata" (cd build && python references.py) # pandoc settings -CSL_PATH=https://github.com/citation-style-language/styles/raw/master/dependent/journal-of-the-royal-society-interface.csl +CSL_PATH=references/deep-review.csl BIBLIOGRAPHY_PATH=references/generated/bibliography.json INPUT_PATH=references/generated/all-sections.md From b7e1b5ad59ff5dff62f757b8bfa071f5aef95a21 Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Wed, 24 May 2017 14:23:54 -0400 Subject: [PATCH 078/216] Update DOI resolution URLs in citeproc --- build/metadata.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/build/metadata.py b/build/metadata.py index 279f569..7438725 100644 --- a/build/metadata.py +++ b/build/metadata.py @@ -35,7 +35,12 @@ def get_doi_citeproc(doi): 'Accept': 'application/vnd.citationstyles.csl+json', } response = requests.get(url, headers=header) - return response.json() + citeproc = response.json() + # Upgrade to preferred formatting in DOI resolution URLs + if 'URL' in citeproc: + pattern = re.compile(r'^(https?://d?x?\.?)doi\.org/') + citeproc['URL'] = pattern.sub('https://doi.org/', citeproc['URL']) + return citeproc def get_pubmed_citeproc(pubmed_id): From ccaa8525effefc0df61634b2db6e7b13c7d11b82 Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Wed, 24 May 2017 14:28:57 -0400 Subject: [PATCH 079/216] Add URL attribute to PubMed citeproc --- build/metadata.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/build/metadata.py b/build/metadata.py index 7438725..a46f562 100644 --- a/build/metadata.py +++ b/build/metadata.py @@ -57,7 +57,9 @@ def get_pubmed_citeproc(pubmed_id): } url = 'https://www.ncbi.nlm.nih.gov/pmc/utils/ctxp' response = requests.get(url, params) - return response.json() + citeproc = response.json() + citeproc['URL'] = f'https://www.ncbi.nlm.nih.gov/pubmed/{pubmed_id}' + return citeproc def get_url_citeproc_greycite(url): From c7f379878de197d936dffe3b25fea14ac8b24965 Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Thu, 25 May 2017 12:08:55 -0400 Subject: [PATCH 080/216] Upgrade arXiv URLs to use HTTPS (#512) --- build/citations.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/build/citations.py b/build/citations.py index fda8061..91f982b 100644 --- a/build/citations.py +++ b/build/citations.py @@ -190,6 +190,10 @@ def bibtex_passthrough(text, set_id=None): # Set URL as url rather than link attribute if 'link' in entry and 'url' not in entry: entry['url'] = entry.pop('link') + # Upgrade arxiv links to HTTPS + if 'url' in entry: + pattern = re.compile(r'^http://arxiv\.org/') + entry['url'] = pattern.sub('https://arxiv.org/', entry['url']) if set_id is not None: entry['ID'] = set_id return bibtexparser.dumps(bibdb) From 2869f1143fb11d3b3c63388acde2b5afde28d6e5 Mon Sep 17 00:00:00 2001 From: Casey Greene Date: Mon, 29 May 2017 13:54:43 -0400 Subject: [PATCH 081/216] Update README to note new manuscript status (#521) * note submitted * Add bioRxiv link --- README.md | 34 +++++++++++++--------------------- 1 file changed, 13 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index fa7819c..a38bc28 100644 --- a/README.md +++ b/README.md @@ -2,32 +2,24 @@ [![Code Climate](https://codeclimate.com/github/greenelab/deep-review/badges/gpa.svg)](https://codeclimate.com/github/greenelab/deep-review) -We have the opportunity to write a headline review for *Journal of the Royal -Society Interface* on a topic overlapping the computer and life sciences in the -area of systems pharmacology. We are hitting the home stretch towards getting -this paper submitted. Consequently, we have established the following -guidelines: - -* April 24: Any empty section, even if in the outline, gets dropped due to lack - of interest. -* May 1: Any section not merged & ready for revisions gets dropped due to lack - of completion. Invitations go out to contributing authors to approve the - manuscript. -* May 1 - 14: Revisions, flow, and clarity. -* Final edits and author approval. Authors must approve by [Noon Eastern - Time](https://time.is/ET) on May 17 to be included as co-authors. File a pull - request to add the appropriate line to `author_contact_approval.md` - -At this stage it is ideal if **each pull request touches no more than one -subsection** as this minimizes the number of people who will need to sign off on -changes. The most current version of the `master` branch is built by continuous integration and [available via gh-pages](https://greenelab.github.io/deep-review/). To see what's incoming, check the [open pull requests](https://github.com/greenelab/deep-review/pulls). -If you want to claim a section (time is short - see timeline above!) issue number -[188](https://github.com/greenelab/deep-review/issues/188) is the place to do it. + +## Current Status - Submitted + +We have submitted the manuscript to the journal. A preprint is available at +[bioRxiv](https://doi.org/10.1101/142760). We are still accepting contributions +to improve this work. Feel free to create an issue, contribute some text via a +pull request, and pitch in. Authorship criteria remain the same. + +## More about the manuscript. + +We have the opportunity to write a headline review for *Journal of the Royal +Society Interface* on a topic overlapping the computer and life sciences in the +area of systems pharmacology. > A Headline Review is one in a short, targeted series of high-level reviews within a particular topic of a burgeoning research area. We encourage authors to From 6a3c3d389ee94fc88ddedaec20a0ffeaa1f8dd5b Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Thu, 1 Jun 2017 17:01:31 -0400 Subject: [PATCH 082/216] Add Google Analytics to HTML output (#525) Closes https://github.com/greenelab/deep-review/issues/524 --- build/assets/analytics.js | 11 +++++++++++ build/build.sh | 1 + 2 files changed, 12 insertions(+) create mode 100644 build/assets/analytics.js diff --git a/build/assets/analytics.js b/build/assets/analytics.js new file mode 100644 index 0000000..f8ed6e3 --- /dev/null +++ b/build/assets/analytics.js @@ -0,0 +1,11 @@ + + + \ No newline at end of file diff --git a/build/build.sh b/build/build.sh index 04e5eaf..465bb5f 100644 --- a/build/build.sh +++ b/build/build.sh @@ -21,6 +21,7 @@ pandoc --verbose \ --csl=$CSL_PATH \ --metadata link-citations=true \ --css=github-pandoc.css \ + --include-in-header=build/assets/analytics.js \ --output=output/index.html \ $INPUT_PATH From d36e3415295c048b2a79943c1ecf66ef7c8df2a4 Mon Sep 17 00:00:00 2001 From: Robert Gieseke Date: Wed, 7 Jun 2017 22:06:07 +0200 Subject: [PATCH 083/216] Add anchor links to headers in the HTML version (#534) Add anchor links to headers in the HTML version The MIT-licensed AnchorJS library is used: `build/assets/anchors.js` derives from https://github.com/bryanbraun/anchorjs/blob/bf22c0f9832ca160b361a3bbc4112528bc84b460/anchor.min.js --- build/assets/anchors.js | 12 ++++++++++++ build/build.sh | 1 + 2 files changed, 13 insertions(+) create mode 100644 build/assets/anchors.js diff --git a/build/assets/anchors.js b/build/assets/anchors.js new file mode 100644 index 0000000..3d8ed07 --- /dev/null +++ b/build/assets/anchors.js @@ -0,0 +1,12 @@ + diff --git a/build/build.sh b/build/build.sh index 465bb5f..07bd701 100644 --- a/build/build.sh +++ b/build/build.sh @@ -22,6 +22,7 @@ pandoc --verbose \ --metadata link-citations=true \ --css=github-pandoc.css \ --include-in-header=build/assets/analytics.js \ + --include-after-body=build/assets/anchors.js \ --output=output/index.html \ $INPUT_PATH From c3580b3323380af1f44cbc044e074ac3ac257745 Mon Sep 17 00:00:00 2001 From: Anthony Gitter Date: Mon, 12 Jun 2017 16:27:26 -0500 Subject: [PATCH 084/216] Convert author table from md to tsv (#541) * Convert author table from md to tsv * Convert tsv from Unicode to UTF-8 * Machine readable column headers --- author_contact_approve.tsv | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 author_contact_approve.tsv diff --git a/author_contact_approve.tsv b/author_contact_approve.tsv new file mode 100644 index 0000000..e57af8d --- /dev/null +++ b/author_contact_approve.tsv @@ -0,0 +1,28 @@ +github_handle full_name orcid affiliation email approve funding +cgreene Casey S. Greene 0000-0001-8713-9213 Department of Systems Pharmacology and Translational Therapeutics, Perelman School of Medicine, University of Pennsylvania, Philadelphia, PA csgreene@upenn.edu Yes Gordon and Betty Moore Foundation GBMF 4552 +j3xugit Jinbo Xu 0000-0001-7111-4839 Toyota Technological Institute at Chicago, Chicago, IL jinboxu@gmail.com Yes NIH R01GM089753, NSF 1564955 +gailrosen Gail L. Rosen 0000-0003-1763-5750 Ecological and Evolutionary Signal-processing and Informatics Laboratory, Department of Electrical and Computer Engineering, Drexel University, Philadelphia, PA gailr@ece.drexel.edu Yes NSF 1245632 +davharris David J. Harris 0000-0003-3332-9307 Department of Wildlife Ecology and Conservation, University of Florida, Gainesville, FL dave@harris-research.me Yes Gordon and Betty Moore Foundation GBMF 4563 +bdo311 Brian T. Do 0000-0003-4992-2623 Harvard Medical School, Boston, MA brian_do@hms.harvard.edu Yes NIH T32GM007753 +gwaygenomics Gregory P. Way 0000-0002-0503-9348 Department of Systems Pharmacology and Translational Therapeutics, Perelman School of Medicine, University of Pennsylvania, Philadelphia, PA gregway@upenn.edu Yes +alxndrkalinin Alexandr A. Kalinin 0000-0003-4563-3226 Department of Computational Medicine and Bioinformatics, University of Michigan Medical School, Ann Arbor, MI akalinin@umich.edu Yes +agitter Anthony Gitter 0000-0002-5324-9833 Department of Biostatistics and Medical Informatics, University of Wisconsin-Madison and Morgridge Institute for Research, Madison, WI gitter@biostat.wisc.edu Yes NIH U54AI117924 +brettbj Brett K. Beaulieu-Jones 0000-0002-6700-1468 Genomics and Computational Biology Graduate Group, Perelman School of Medicine, University of Pennsylvania, Philadelphia, PA brettbe@upenn.edu Yes NIH R01AI116794 +blengerich Benjamin J. Lengerich 0000-0001-8690-9554 Computational Biology Department, School of Computer Science, Carnegie Mellon University, Pittsburgh, PA blengeri@cs.cmu.edu Yes +agapow Paul-Michael Agapow 0000-0003-1126-1479 Data Science Institute, Imperial College London, London, United Kingdom p.agapow@imperial.ac.uk Yes +enricoferrero Enrico Ferrero 0000-0002-8362-100X Computational Biology and Stats, Target Sciences, GlaxoSmithKline, Stevenage, United Kingdom enrico.x.ferrero@gsk.com Yes +annecarpenter Anne E. Carpenter 0000-0003-1555-8261 Imaging Platform, Broad Institute of Harvard and MIT, Cambridge, MA anne@broadinstitute.org Yes NIH R01GM089652 +evancofer Evan M. Cofer 0000-0003-3877-0433 Department of Computer Science, Trinity University, San Antonio, TX ecofer@princeton.edu Yes NSF 1531594 +DaveDeCaprio Dave DeCaprio 0000-0001-8931-9461 ClosedLoop.ai, Austin, TX daved@alum.mit.edu Yes +akundaje Anshul Kundaje 0000-0003-3084-2287 Department of Genetics and Department of Computer Science, Stanford University, Stanford, CA akundaje@stanford.edu Yes NIH DP2GM123485 +sw1 Stephen Woloszynek 0000-0003-0568-298X Ecological and Evolutionary Signal-processing and Informatics Laboratory, Department of Electrical and Computer Engineering, Drexel University, Philadelphia, PA sw424@drexel.edu Yes +yfpeng Yifan Peng 0000-0001-9309-8331 National Center for Biotechnology Information and National Library of Medicine, National Institutes of Health, Bethesda, MD yifan.peng@nih.gov Yes National Institutes of Health Intramural Research Program and National Library of Medicine +dhimmel Daniel S. Himmelstein 0000-0002-3012-7446 Department of Systems Pharmacology and Translational Therapeutics, Perelman School of Medicine, University of Pennsylvania, Philadelphia, PA daniel.himmelstein@gmail.com Yes Gordon and Betty Moore Foundation GBMF 4552 +jacklanchantin Jack Lanchantin 0000-0003-0811-0944 Department of Computer Science, University of Virginia, Charlottesville, VA jjl5sw@virginia.edu Yes +qiyanjun Yanjun Qi 0000-0002-5796-7453 Department of Computer Science, University of Virginia, Charlottesville, VA yq2h@virginia.edu Yes +mrwns Marwin H. S. Segler 0000-0001-8008-0546 Institute of Organic Chemistry, Westfälische Wilhelms-Universität Münster, Münster, Germany marwin.segler@uni-muenster.de Yes +jisraeli Johnny Israeli 0000-0003-1633-5780 Biophysics Program, Stanford University, Stanford, CA jisraeli@stanford.edu Yes +xieconnect Wei Xie 0000-0002-1871-6846 Electrical Engineering and Computer Science, Vanderbilt University, Nashville, TN wei.xie@vanderbilt.edu Yes +AvantiShri Avanti Shrikumar 0000-0002-6443-4671 Department of Computer Science, Stanford University, Stanford, CA avanti@stanford.edu Yes +laurakwiley Laura K. Wiley 0000-0001-6681-9754 Division of Biomedical Informatics and Personalized Medicine, University of Colorado School of Medicine, Aurora, CO laura.wiley@ucdenver.edu Yes +traversc Travers Ching 0000-0002-5577-3516 Molecular Biosciences and Bioengineering Graduate Program, University of Hawaii at Manoa, Honolulu, HI traversc@gmail.com Yes From 0750b8c756678675d076ae577f343b293579545f Mon Sep 17 00:00:00 2001 From: Anthony Gitter Date: Sat, 17 Jun 2017 14:25:29 -0500 Subject: [PATCH 085/216] Update build notes (#545) * Update build notes * Add note about Linux requirement --- build/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/build/README.md b/build/README.md index ea6430a..16133ca 100644 --- a/build/README.md +++ b/build/README.md @@ -5,10 +5,11 @@ ## Environment -Install the [conda](https://conda.io) environment specified in [`environment.yml`](../environment.yml) by running: +Install the [conda](https://conda.io) environment specified in [`environment.yml`](environment.yml) by running: ```sh conda env create --file environment.yml ``` -Activate with `source activate deep-review`. +Activate with `source activate deepreview`. +Currently, the environment requires Linux. From e055c18308dd3c2b7a8eb7fa521cd9fca24a963a Mon Sep 17 00:00:00 2001 From: Anthony Gitter Date: Sun, 18 Jun 2017 06:02:48 -0500 Subject: [PATCH 086/216] Template for author counts (#546) * Test template with hard-coded author count * Move hard-coded author counts to authors.py * Dynamically get author count for template * Rename authors file and switch to OrderedDict --- author_contact_approve.tsv | 28 ---------------------------- build/authors.py | 18 ++++++++++++++++++ build/references.py | 6 ++++++ 3 files changed, 24 insertions(+), 28 deletions(-) delete mode 100644 author_contact_approve.tsv create mode 100644 build/authors.py diff --git a/author_contact_approve.tsv b/author_contact_approve.tsv deleted file mode 100644 index e57af8d..0000000 --- a/author_contact_approve.tsv +++ /dev/null @@ -1,28 +0,0 @@ -github_handle full_name orcid affiliation email approve funding -cgreene Casey S. Greene 0000-0001-8713-9213 Department of Systems Pharmacology and Translational Therapeutics, Perelman School of Medicine, University of Pennsylvania, Philadelphia, PA csgreene@upenn.edu Yes Gordon and Betty Moore Foundation GBMF 4552 -j3xugit Jinbo Xu 0000-0001-7111-4839 Toyota Technological Institute at Chicago, Chicago, IL jinboxu@gmail.com Yes NIH R01GM089753, NSF 1564955 -gailrosen Gail L. Rosen 0000-0003-1763-5750 Ecological and Evolutionary Signal-processing and Informatics Laboratory, Department of Electrical and Computer Engineering, Drexel University, Philadelphia, PA gailr@ece.drexel.edu Yes NSF 1245632 -davharris David J. Harris 0000-0003-3332-9307 Department of Wildlife Ecology and Conservation, University of Florida, Gainesville, FL dave@harris-research.me Yes Gordon and Betty Moore Foundation GBMF 4563 -bdo311 Brian T. Do 0000-0003-4992-2623 Harvard Medical School, Boston, MA brian_do@hms.harvard.edu Yes NIH T32GM007753 -gwaygenomics Gregory P. Way 0000-0002-0503-9348 Department of Systems Pharmacology and Translational Therapeutics, Perelman School of Medicine, University of Pennsylvania, Philadelphia, PA gregway@upenn.edu Yes -alxndrkalinin Alexandr A. Kalinin 0000-0003-4563-3226 Department of Computational Medicine and Bioinformatics, University of Michigan Medical School, Ann Arbor, MI akalinin@umich.edu Yes -agitter Anthony Gitter 0000-0002-5324-9833 Department of Biostatistics and Medical Informatics, University of Wisconsin-Madison and Morgridge Institute for Research, Madison, WI gitter@biostat.wisc.edu Yes NIH U54AI117924 -brettbj Brett K. Beaulieu-Jones 0000-0002-6700-1468 Genomics and Computational Biology Graduate Group, Perelman School of Medicine, University of Pennsylvania, Philadelphia, PA brettbe@upenn.edu Yes NIH R01AI116794 -blengerich Benjamin J. Lengerich 0000-0001-8690-9554 Computational Biology Department, School of Computer Science, Carnegie Mellon University, Pittsburgh, PA blengeri@cs.cmu.edu Yes -agapow Paul-Michael Agapow 0000-0003-1126-1479 Data Science Institute, Imperial College London, London, United Kingdom p.agapow@imperial.ac.uk Yes -enricoferrero Enrico Ferrero 0000-0002-8362-100X Computational Biology and Stats, Target Sciences, GlaxoSmithKline, Stevenage, United Kingdom enrico.x.ferrero@gsk.com Yes -annecarpenter Anne E. Carpenter 0000-0003-1555-8261 Imaging Platform, Broad Institute of Harvard and MIT, Cambridge, MA anne@broadinstitute.org Yes NIH R01GM089652 -evancofer Evan M. Cofer 0000-0003-3877-0433 Department of Computer Science, Trinity University, San Antonio, TX ecofer@princeton.edu Yes NSF 1531594 -DaveDeCaprio Dave DeCaprio 0000-0001-8931-9461 ClosedLoop.ai, Austin, TX daved@alum.mit.edu Yes -akundaje Anshul Kundaje 0000-0003-3084-2287 Department of Genetics and Department of Computer Science, Stanford University, Stanford, CA akundaje@stanford.edu Yes NIH DP2GM123485 -sw1 Stephen Woloszynek 0000-0003-0568-298X Ecological and Evolutionary Signal-processing and Informatics Laboratory, Department of Electrical and Computer Engineering, Drexel University, Philadelphia, PA sw424@drexel.edu Yes -yfpeng Yifan Peng 0000-0001-9309-8331 National Center for Biotechnology Information and National Library of Medicine, National Institutes of Health, Bethesda, MD yifan.peng@nih.gov Yes National Institutes of Health Intramural Research Program and National Library of Medicine -dhimmel Daniel S. Himmelstein 0000-0002-3012-7446 Department of Systems Pharmacology and Translational Therapeutics, Perelman School of Medicine, University of Pennsylvania, Philadelphia, PA daniel.himmelstein@gmail.com Yes Gordon and Betty Moore Foundation GBMF 4552 -jacklanchantin Jack Lanchantin 0000-0003-0811-0944 Department of Computer Science, University of Virginia, Charlottesville, VA jjl5sw@virginia.edu Yes -qiyanjun Yanjun Qi 0000-0002-5796-7453 Department of Computer Science, University of Virginia, Charlottesville, VA yq2h@virginia.edu Yes -mrwns Marwin H. S. Segler 0000-0001-8008-0546 Institute of Organic Chemistry, Westfälische Wilhelms-Universität Münster, Münster, Germany marwin.segler@uni-muenster.de Yes -jisraeli Johnny Israeli 0000-0003-1633-5780 Biophysics Program, Stanford University, Stanford, CA jisraeli@stanford.edu Yes -xieconnect Wei Xie 0000-0002-1871-6846 Electrical Engineering and Computer Science, Vanderbilt University, Nashville, TN wei.xie@vanderbilt.edu Yes -AvantiShri Avanti Shrikumar 0000-0002-6443-4671 Department of Computer Science, Stanford University, Stanford, CA avanti@stanford.edu Yes -laurakwiley Laura K. Wiley 0000-0001-6681-9754 Division of Biomedical Informatics and Personalized Medicine, University of Colorado School of Medicine, Aurora, CO laura.wiley@ucdenver.edu Yes -traversc Travers Ching 0000-0002-5577-3516 Molecular Biosciences and Bioengineering Graduate Program, University of Hawaii at Manoa, Honolulu, HI traversc@gmail.com Yes diff --git a/build/authors.py b/build/authors.py new file mode 100644 index 0000000..997bb23 --- /dev/null +++ b/build/authors.py @@ -0,0 +1,18 @@ +""" +Process author table +""" + +import collections +import pandas + + +def get_author_info(author_file): + """ + Load the author table and return a dict with author information such as + the number of authors. + """ + author_df = pandas.read_table(author_file) + + author_info = collections.OrderedDict() + author_info['count'] = len(author_df) + return author_info diff --git a/build/references.py b/build/references.py index d9e8f2b..6910ec5 100644 --- a/build/references.py +++ b/build/references.py @@ -25,6 +25,8 @@ validate_reference, ) +from authors import get_author_info + # Run only as a script assert __name__ == '__main__' @@ -150,6 +152,10 @@ def get_standard_citatation(citation, cache, override): print('References by type:') print(ref_counts) +# Author table information +authors_path = pathlib.Path('../authors.tsv') +stats['authors'] = get_author_info(authors_path) + with gen_dir.joinpath('stats.json').open('wt') as write_file: json.dump(stats, write_file, indent=2) From bf9de06cd6a5536923eb541bc6f26c6954d1bd36 Mon Sep 17 00:00:00 2001 From: Robert Gieseke Date: Wed, 21 Jun 2017 16:45:22 +0200 Subject: [PATCH 087/216] Fix typo (#552) --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f93b563..030a9a8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -36,7 +36,7 @@ additional feedback. ## Markdown The paper will be written using markdown. Markdown files use the `.md` -extension. Check out the [CommarkMark Help](http://commonmark.org/help/) page +extension. Check out the [CommonMark Help](http://commonmark.org/help/) page for an introduction to formatting options provided by markdown. In addition, to standard markdown features, we support [markdown tables](https://help.github.com/articles/organizing-information-with-tables/ From 1a2afbb2ba1938b89beb456afaf6f83e2bc16d88 Mon Sep 17 00:00:00 2001 From: Anthony Gitter Date: Wed, 21 Jun 2017 10:18:33 -0500 Subject: [PATCH 088/216] Templates for author list, affiliations, and funding (#550) * Temporarily add exported HTML manuscript to support subsequent diff * Derive author initials and confirm authors approve manuscript * Standardize funders in author table * Further standardize funding information in author table * Add functions to format funding information for templates * Add templates for funding information * Add predetermined author order to prepare for additional templating * Generate affiliation numbers * Refactor affiliation code and add annotation to first author * Add corresponding author annotation * Better support for multiple middle initials * Remove trailing whitespace * Add template for corresponding author contact information * Replace defaultdict with setdefault * Sort author table and remove author ordering column * Switch to OrderedDict for funding information * Separate author name, affiliation, and symbol for template * Improve code style * Format affiliations in jinja2 * Ignore and remove exported HTML manuscipt --- build/authors.py | 130 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 128 insertions(+), 2 deletions(-) diff --git a/build/authors.py b/build/authors.py index 997bb23..1408ca3 100644 --- a/build/authors.py +++ b/build/authors.py @@ -2,9 +2,119 @@ Process author table """ -import collections +from collections import OrderedDict + import pandas +import numpy as np + + +def initials(full_name): + """ + Return author intials in the form F.L. for First Last. Accomodates an + aribtrary number of whitespace-separated strings. Converts periods to + whitespace so that multiple middle initials are retained. + """ + full_name = full_name.replace('.', ' ') + name_tokens = full_name.strip().split() + # Keeps only the first part of hyphenated names + name_tokens = [name[0] + '.' for name in name_tokens] + return ''.join(name_tokens) + +def format_list(tokens): + """ + Format a list of string tokens such that a pair [X, Y] is formatted 'X and + Y' and a list [X, Y, Z] becomes 'X, Y, and Z'. Single tokens are returned + as-is. + """ + if len(tokens) <= 2: + return ' and '.join(tokens) + else: + tokens[-1] = 'and ' + tokens[-1] + return ', '.join(tokens) + +def get_funding(author_df): + """ + Returns a dictionary mapping funders ('GBMF', 'NIH', 'NSF', 'OTHER') to + award and author text. An author may have more than one funding + source and multiple authors may have the same funding source. + """ + # Initially create a map from funders to dictionaries that map an award to + # the authors supported by that award + # funding_info data structure has the form + # {funder1: {award1: [author1], award2: [author2, author3]}} + funding_info = OrderedDict() + for (_, row) in author_df.dropna(subset=['funding']).iterrows(): + for grant in row['funding'].split(','): + # Separate into the funder and the award + grant_tokens = grant.strip().split(None, 1) + funder_map = funding_info.setdefault(grant_tokens[0], dict()) + funder_map.setdefault(grant_tokens[1], list()).append(row['initials']) + + formatted_info = OrderedDict() + for funder in funding_info.keys(): + # The map from awards to author lists for a single funder + funder_map = funding_info[funder] + funder_awards = list() + for award in sorted(funder_map.keys()): + authors = sorted(funder_map[award]) + funder_awards.append(f'{award} ({format_list(authors)})') + formatted_info[funder] = format_list(funder_awards) + return formatted_info + +def get_affiliations(author_df): + """ + Takes an author data frame in which authors have already been ordered and + creates affiliation mappings. Each unique affiliation is assigned an + index based on the order in which it appears in the ordered author list. + Complete affiliation strings are used to determine affiliation uniqueness + even if on string contains multiple departments or institutions. + """ + # Each author is a dict with a name, affiliation index, and symbol that + # indicates corresponding or other special notes + authors = list() + affiliations = list() + corresponding = list() + affiliation_map = dict() + affiliation_counter = 1 + + for (_, row) in author_df.iterrows(): + name = row['full_name'] + affiliation = row['affiliation'] + + # Lookup the affiliation index or create it + # Only list each affiliation once + if not affiliation in affiliation_map: + affiliation_map[affiliation] = affiliation_counter + affiliation_info = OrderedDict() + affiliation_info['index'] = affiliation_counter + affiliation_info['institution'] = affiliation + affiliations.append(affiliation_info) + affiliation_counter += 1 + index = affiliation_map[affiliation] + + # Add the affiliation superscript + # Assumes a single affiliation per author (which could contain multiple + # institutions) but could be generalized + author = OrderedDict() + author['name'] = name + author['affiliation'] = index + if pandas.notnull(row['corresponding']) and row['corresponding'].lower() == 'yes': + author['symbol'] = ',†' + corresponding.append(f'{row["email"]} ({row["initials"]})') + else: + author['symbol'] = '' + authors.append(author) + + # Add special annotation for the first listed author + first = authors[0] + first['symbol'] = first['symbol'] + ',*' + + affiliation_info = OrderedDict() + affiliation_info['authors'] = authors + affiliation_info['affiliations'] = affiliations + affiliation_info['corresponding'] = format_list(corresponding) + return affiliation_info def get_author_info(author_file): """ @@ -13,6 +123,22 @@ def get_author_info(author_file): """ author_df = pandas.read_table(author_file) - author_info = collections.OrderedDict() + # Add author initials + author_df['initials'] = author_df['full_name'].apply(initials) + + # Avoid initial collisions, will need a more general way to refer to + # authors if they occur + assert len(author_df['initials'].unique()) == len(author_df), \ + 'All authors must have unique initials' + assert np.all(author_df['approve'].str.match('Yes', case=False)), \ + 'All contributors must approve of the manuscript before being ' + \ + 'added as authors' + + author_info = OrderedDict() author_info['count'] = len(author_df) + + # Add funding, author order, and affiliation information + author_info.update(get_funding(author_df)) + author_info.update(get_affiliations(author_df)) + return author_info From 59991680e68d3af315203891945f8737f5559d1b Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Thu, 29 Jun 2017 13:53:24 -0400 Subject: [PATCH 089/216] Migrate from deep-review to manubot-rootstock History proceeding this commit was generated from https://github.com/greenelab/deep-review/commit/daef821cc87f264063cb0f32e79d284a756d56bc by running: git filter-branch \ --index-filter 'git rm --cached --ignore-unmatch -r authors references sections .codeclimate.yml author_contact_approve.md authors.tsv'\ --prune-empty -- --all --- .gitignore | 2 +- .travis.yml | 2 +- CONTRIBUTING.md | 68 +++++----------------- README.md | 115 +++++--------------------------------- build/README.md | 4 +- build/assets/analytics.js | 11 +--- build/assets/anchors.js | 5 +- build/build.sh | 4 +- build/environment.yml | 2 +- ci/README.md | 4 +- ci/deploy.sh | 10 ++-- output/README.md | 6 +- 12 files changed, 50 insertions(+), 183 deletions(-) diff --git a/.gitignore b/.gitignore index 0f1030f..7caeb02 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ # Generated manuscript output files output/index.html -output/deep-review.pdf +output/manuscript.pdf output/*.ots # Generated reference files diff --git a/.travis.yml b/.travis.yml index ba6e2a5..7ee26fe 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,7 +15,7 @@ before_install: - conda info --all install: - conda env create --quiet --file build/environment.yml - - source activate deepreview + - source activate manubot script: - sh build/build.sh cache: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 030a9a8..c1e4b8f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,64 +1,22 @@ -# Contribution guidelines for the deep-review repository - -## Issues - -We'll use issues for discussion of papers, section outlines, and other -structural components of the paper. - -## Pull requests - -Contributions to the article operate on a pull request model. We expect -participants to actively review pull requests. We'd love to have you ask -questions, clarify points, and jump in and edit the text. - -## Authorship - -What qualifies as authorship? We use the [ICJME Guidelines](http://www.icmje.org/recommendations/browse/roles-and-responsibilities/defining-the-role-of-authors-and-contributors.html). -We expect authors to contribute to the overall design by participating in -issues, to contribute to the text by contributing sections and/or revisions -to sections through pull requests. It is important to note that, for authorship, -these should be substantial intellectual contributions. - -## Peer review - -All pull requests will undergo peer review. Participants in this project should -review the changes. They should suggest modifications or, potentially, directly -edit the pull request to make suggested changes. Reviewers can note approval by -commenting ":+1:", "looks good to me", or "approved". As a reviewer, it's -helpful to note the type of review you performed: did you read cited literature, -look over the text in detail, or are you just supporting the concept? - -Before a repository maintainer merges a pull request, there must be at least -one affirmative review. If there is any unaddressed criticism or disapproval, -a repository maintainer will determine how to proceed and may wait for -additional feedback. +# Manuscript contribution guidelines ## Markdown -The paper will be written using markdown. Markdown files use the `.md` -extension. Check out the [CommonMark Help](http://commonmark.org/help/) page -for an introduction to formatting options provided by markdown. In addition, to -standard markdown features, we support [markdown -tables](https://help.github.com/articles/organizing-information-with-tables/ -"GitHub Help: Organizing information with tables") and a custom citation syntax. -Check out [Tables Generator](http://www.tablesgenerator.com/markdown_tables) for -creating markdown tables. +The paper will be written using markdown. Markdown files use the `.md` extension. +Check out the [CommonMark Help](http://commonmark.org/help/) page for an introduction to formatting options provided by markdown. +In addition, to standard markdown features, we support [markdown tables](https://help.github.com/articles/organizing-information-with-tables/ "GitHub Help: Organizing information with tables") and a custom citation syntax. +Check out [Tables Generator](http://www.tablesgenerator.com/markdown_tables) for creating markdown tables. The custom citation guidelines are as follows: -1. Always use a DOI for the version of record if available. Cite DOIs like - `[@doi:10.15363/thinklab.4]`. -2. If the version of record doesn't have a DOI but does have a PubMed ID, cite - like `[@pmid:26158728]`. +1. Always use a DOI for the version of record if available. + Cite DOIs like `[@doi:10.15363/thinklab.4]`. +2. If the version of record doesn't have a DOI but does have a PubMed ID, cite like `[@pmid:26158728]`. 3. If the article is an _arXiv_ preprint, cite like `[@arxiv:1508.06576]`. -4. If and only if the article has none of the above, cite with the URL like - `[@url:http://openreview.net/pdf?id=Sk-oDY9ge]`. +4. If and only if the article has none of the above, cite with the URL like `[@url:http://openreview.net/pdf?id=Sk-oDY9ge]`. -You cite multiple items at once like `[@doi:10.15363/thinklab.4 @pmid:26158728 -@arxiv:1508.06576]`. +You cite multiple items at once like `[@doi:10.15363/thinklab.4 @pmid:26158728 @arxiv:1508.06576]`. -For improved readability, you can (but are not required to) use a tag. For -example, you can add a reference to the tag `Zhou2015_deep_sea` using the -following syntax: `[@tag:Zhou2015_deep_sea]`. If you add references that use -tags, make sure to add those tags and their corresponding citations to -[`references/tags.tsv`](references/tags.tsv). +The system also supports tags, which may be helpful when a single reference is cited many times. +For example, you can add a reference to the tag `study_x` using the following syntax: `[@tag:study_x]`. +If you add references that use tags, make sure to add those tags and their corresponding citations to [`references/tags.tsv`](references/tags.tsv). diff --git a/README.md b/README.md index a38bc28..d0696fe 100644 --- a/README.md +++ b/README.md @@ -1,113 +1,26 @@ -# Opportunities and obstacles for deep learning in biology and medicine - -[![Code Climate](https://codeclimate.com/github/greenelab/deep-review/badges/gpa.svg)](https://codeclimate.com/github/greenelab/deep-review) - - -The most current version of the `master` branch is built by continuous integration -and [available via gh-pages](https://greenelab.github.io/deep-review/). To see -what's incoming, check the -[open pull requests](https://github.com/greenelab/deep-review/pulls). - -## Current Status - Submitted - -We have submitted the manuscript to the journal. A preprint is available at -[bioRxiv](https://doi.org/10.1101/142760). We are still accepting contributions -to improve this work. Feel free to create an issue, contribute some text via a -pull request, and pitch in. Authorship criteria remain the same. - -## More about the manuscript. - -We have the opportunity to write a headline review for *Journal of the Royal -Society Interface* on a topic overlapping the computer and life sciences in the -area of systems pharmacology. - -> A Headline Review is one in a short, targeted series of high-level reviews -within a particular topic of a burgeoning research area. We encourage authors to -write in a style that opens the door to a broad range of readers working at the -physical sciences - life sciences interface. We intend the reviews to address -critical developments in an area of cross-disciplinary research and, when -possible, to place such research in a broader context. This is not a place for -comprehensive literature surveys. -> -> We do encourage you to speculate in an informed way, and to be topical and -provocative about the subject without worrying unduly about space, (the -provisional target length is 8 -12,000 words). Please think of this as an -article which will be a landmark in your area, and will come to be considered as -a classic paper of the literature. - -I was recently inspired by [Harold Pimentel's](https://github.com/pimentel) -crowd-sourced [collection of deep learning -papers](https://github.com/pimentel/deep_learning_papers). Instead of having one -individual write this, I thought that this invitation provided a wonderful -opportunity to take advantage of the wisdom of crowds to bring a team together -around this topic. - -This repository provides a home for the paper. We'll operate on a pull request -model. Anyone whose contributions meet the ICJME standards of authorship will be -included as an author on the manuscript. I can't guarantee that it will be -accepted, but I look forward to trying this approach out. - -## Status Report on 12/21 - -We are now actively writing the review. Markdown files can be found in the -`sections/` folder. Please claim a section, create a fork, and contribute that -section back via a pull request. To see what a pull request into the paper -entails, check out [PR #147](https://github.com/greenelab/deep-review/pull/147) -from [@evancofer](https://github.com/evancofer). - -## Status Report on 10/26 - -We are now actively outlining the review sections and will begin writing -soon. The goal is to have a complete draft in about a month. The action items -from the 8/25 status report below are still applicable. In addition, you can: - -1. Sign up to write in [#116](https://github.com/greenelab/deep-review/issues/116) - and share which sections you are most interested in. We are in need of - experts in biomedical imaging applications in particular. -2. Review the stubs in the `sections` subdirectory and respond to the prompts - with a pull request. -3. Outline sections that do not have stubs with a pull request and discuss - them with other co-authors in the pull request comments. - -## Status Report on 8/25 - -Over the first three weeks of this project, we've developed an initial guiding -question; collaboratively read, summarized, and discussed existing literature -through github issues; and we're now refining our guiding question. If you want -to begin to contribute to this review now, there are a few steps that you may -want to take to get up to speed quickly. - -1. Read through issue [#2](https://github.com/greenelab/deep-review/issues/2). - This will give an idea of what our perspective was as we were starting out - and planning to read papers. -2. Peruse some of the papers that the group has already reviewed, and take a - look at the review. Fill in gaps that you see in the short summary/discussion - of the paper. -3. Choose some papers in an area that you care about, review them, and summarize - them. -4. Dive into [#88](https://github.com/greenelab/deep-review/issues/88) and help - us to further refine the specific question that we're going to deal with in - this review. - -In about a week, we plan to move into the phase where we start to vigorously -argue about the answer to the question that we coalesce on with -[#88](https://github.com/greenelab/deep-review/issues/88) for each area that the -review will cover. +# manubot-rootstock + +A rootstock (or rhizome) is "a plant onto which another variety is grafted". +This repository plays the role of a botanical trunk. +Fork it and a manuscript will grow. + +The most current version of the `master` branch is built by continuous integration and [available via gh-pages](https://greenelab.github.io/manubot-rootstock/). +To see what's incoming, check the [open pull requests](https://github.com/greenelab/manubot-rootstock/pulls). ## Continuous Integration -[![Build Status](https://travis-ci.org/greenelab/deep-review.svg?branch=master)](https://travis-ci.org/greenelab/deep-review) +[![Build Status](https://travis-ci.org/greenelab/manubot-rootstock.svg?branch=master)](https://travis-ci.org/greenelab/manubot-rootstock) When you make a pull request, Travis CI will test whether your changes break the build process to generate the formatted manuscript. The build process aims to detect common errors, such as invalid references. If your build fails, see the Travis CI logs for the cause of failure and revise your pull request accordingly. -When a pull request is merged, Travis CI performs the build and writes the results to the [`gh-pages`](https://github.com/greenelab/deep-review/tree/gh-pages) and [`references`](https://github.com/greenelab/deep-review/tree/references) branches. +When a pull request is merged, Travis CI performs the build and writes the results to the [`gh-pages`](https://github.com/greenelab/manubot-rootstock/tree/gh-pages) and [`references`](https://github.com/greenelab/manubot-rootstock/tree/references) branches. The `gh-pages` branch hosts the following URLs: -+ **HTML manuscript** at https://greenelab.github.io/deep-review/
++ **HTML manuscript** at https://greenelab.github.io/manubot-rootstock/
short URL: https://git.io/vytJN -+ **PDF manuscript** at https://greenelab.github.io/deep-review/deep-review.pdf
++ **PDF manuscript** at https://greenelab.github.io/manubot-rootstock/manuscript.pdf
short URL: https://git.io/vytJ5 For continuous integration configuration details, see [`.travis.yml`](.travis.yml). @@ -118,7 +31,7 @@ For continuous integration configuration details, see [`.travis.yml`](.travis.ym [![License: CC0 1.0](https://img.shields.io/badge/License%20Parts-CC0%201.0-lightgrey.svg)](https://creativecommons.org/publicdomain/zero/1.0/) This entirety of this repository is licensed under a CC BY 4.0 License ([`LICENSE.md`](LICENSE.md)), which allows reuse with attribution. -Please attribute by linking to https://github.com/greenelab/deep-review. +Please attribute by linking to https://github.com/greenelab/manubot-rootstock. Since CC BY is not ideal for code and data, certain repository components are also released under the CC0 1.0 public domain dedication ([`LICENSE-CC0.md`](LICENSE-CC0.md)). All files matched by the following blog patterns are dual licensed under CC BY 4.0 and CC0 1.0: @@ -137,4 +50,4 @@ All other files are only available under CC BY 4.0, including: + `*.html` + `*.pdf` -Please open [an issue](https://github.com/greenelab/deep-review/issues) for any question related to licensing. +Please open [an issue](https://github.com/greenelab/manubot-rootstock/issues) for any question related to licensing. diff --git a/build/README.md b/build/README.md index 16133ca..e2f5f40 100644 --- a/build/README.md +++ b/build/README.md @@ -1,4 +1,4 @@ -# Building the deep-review manuscript +# Building the manuscript [`build.sh`](build.sh) builds the repository. `sh build.sh` should be executed from the root directory of the repository. @@ -11,5 +11,5 @@ Install the [conda](https://conda.io) environment specified in [`environment.yml conda env create --file environment.yml ``` -Activate with `source activate deepreview`. +Activate with `source activate manubot`. Currently, the environment requires Linux. diff --git a/build/assets/analytics.js b/build/assets/analytics.js index f8ed6e3..5264bbd 100644 --- a/build/assets/analytics.js +++ b/build/assets/analytics.js @@ -1,11 +1,4 @@ - + - \ No newline at end of file + \ No newline at end of file diff --git a/build/assets/anchors.js b/build/assets/anchors.js index 3d8ed07..5f4777b 100644 --- a/build/assets/anchors.js +++ b/build/assets/anchors.js @@ -1,11 +1,14 @@ diff --git a/build/build.sh b/build/build.sh index b9fbb99..e7c1b1e 100644 --- a/build/build.sh +++ b/build/build.sh @@ -38,6 +38,7 @@ pandoc --verbose \ --css=github-pandoc.css \ --include-in-header=build/assets/analytics.html \ --include-after-body=build/assets/anchors.html \ + --include-after-body=build/assets/hypothesis.html \ --output=output/manuscript.html \ $INPUT_PATH From b030529e234dbac43f88570beab406e4950b1435 Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Tue, 9 Jan 2018 13:23:55 -0500 Subject: [PATCH 167/216] Manubot numbered affiliations upgrade Refs https://github.com/greenelab/manubot/pull/22 --- USAGE.md | 4 +++- build/environment.yml | 2 +- content/00.front-matter.md | 2 +- content/metadata.yaml | 7 +++++-- 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/USAGE.md b/USAGE.md index b867052..39644b0 100644 --- a/USAGE.md +++ b/USAGE.md @@ -135,7 +135,9 @@ initials: DSH # strongly suggested orcid: 0000-0002-3012-7446 # mandatory twitter: dhimmel # optional email: daniel.himmelstein@gmail.com # suggested -affiliations: Department of Systems Pharmacology and Translational Therapeutics, University of Pennsylvania # strongly suggested +affiliations: # as a list, strongly suggested + - Department of Systems Pharmacology and Translational Therapeutics, University of Pennsylvania + - Department of Biological & Medical Informatics, University of California, San Francisco funders: GBMF4552 # optional ``` diff --git a/build/environment.yml b/build/environment.yml index fad3c91..a8ca1b4 100644 --- a/build/environment.yml +++ b/build/environment.yml @@ -14,7 +14,7 @@ dependencies: - pip: - errorhandler==2.0.1 - ghp-import==0.5.5 - - git+https://github.com/greenelab/manubot@855c8491d6e82c88dd126fda901e52c59c78b0d2 + - git+https://github.com/greenelab/manubot@33ff36728e23f3c685f3b00a6a26f19a6fcbf68e - opentimestamps-client==0.5.0 - opentimestamps==0.1.0 - pandoc-eqnos==0.18.1 diff --git a/content/00.front-matter.md b/content/00.front-matter.md index 6219bb6..22dc935 100644 --- a/content/00.front-matter.md +++ b/content/00.front-matter.md @@ -36,7 +36,7 @@ on {{date}}. {%- endif %}
{%- if author.affiliations is defined %} - {{author.affiliations}} + {{author.affiliations | join('; ')}} {%- endif %} {%- if author.funders is defined %} · Funded by {{author.funders}} diff --git a/content/metadata.yaml b/content/metadata.yaml index fbcb6e3..31e6adb 100644 --- a/content/metadata.yaml +++ b/content/metadata.yaml @@ -13,7 +13,8 @@ author_info: orcid: XXXX-XXXX-XXXX-XXXX twitter: johndoe email: john.doe@something.com - affiliations: Department of Something, University of Whatever + affiliations: + - Department of Something, University of Whatever funders: Grant XXXXXXXX - github: janeroe @@ -21,4 +22,6 @@ author_info: initials: JR orcid: XXXX-XXXX-XXXX-XXXX email: jane.roe@whatever.edu - affiliations: Department of Whatever, University of Something + affiliations: + - Department of Something, University of Whatever + - Department of Whatever, University of Something From c31c5d073fde6a0c50d459d8846687fdf6ba1ed3 Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Tue, 9 Jan 2018 13:48:15 -0500 Subject: [PATCH 168/216] MAINT: update packages on 2018-01-09 Update pandoc to 2.1 Reference 2018 development proposal in USAGE.md Remove temporarily broken DOI: https://twitter.com/dhimmel/status/950443969313419264 --- .travis.yml | 2 +- README.md | 4 ++-- USAGE.md | 3 +++ build/README.md | 8 ++++++-- build/environment.yml | 20 ++++++++++---------- content/02.delete-me.md | 2 +- 6 files changed, 23 insertions(+), 16 deletions(-) diff --git a/.travis.yml b/.travis.yml index 979b2bd..2267dab 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,7 @@ branches: only: - master before_install: - - wget https://repo.continuum.io/miniconda/Miniconda3-4.3.30-Linux-x86_64.sh + - wget https://repo.continuum.io/miniconda/Miniconda3-4.3.31-Linux-x86_64.sh --output-document miniconda.sh - bash miniconda.sh -b -p $HOME/miniconda - export PATH="$HOME/miniconda/bin:$PATH" diff --git a/README.md b/README.md index d75e629..7d17415 100644 --- a/README.md +++ b/README.md @@ -41,8 +41,8 @@ To run the Manubot locally, install the [conda](https://conda.io) environment as Then, you can build the manuscript on POSIX systems by running the following commands. ```sh -# Activate the manubot conda environment -source activate manubot +# Activate the manubot conda environment (assumes conda version >= 4.4) +conda activate manubot # Build the manuscript sh build/build.sh diff --git a/USAGE.md b/USAGE.md index 39644b0..9b1b379 100644 --- a/USAGE.md +++ b/USAGE.md @@ -141,6 +141,8 @@ affiliations: # as a list, strongly suggested funders: GBMF4552 # optional ``` +Note that `affiliations` should be a list to allow for multiple affiliations per author. + ## Manubot feedback If you experience any issues with the Manubot or would like to contribute to its source code, please visit [`greenelab/manubot`](https://github.com/greenelab/manubot) or [`greenelab/manubot-rootstock`](https://github.com/greenelab/manubot-rootstock). @@ -156,5 +158,6 @@ For additional examples, check out existing manuscripts that use the Manubot: + The Meta Review ([source](https://github.com/greenelab/meta-review), [manuscript](https://greenelab.github.io/meta-review/)) + The Project Rephetio Manuscript ([source](https://github.com/dhimmel/rephetio-manuscript/), [manuscript](https://dhimmel.github.io/rephetio-manuscript/)) + A Literature Review for Project Planning by David Slochower ([source](https://github.com/slochower/synthetic-motor-literature), [manuscript](https://slochower.github.io/synthetic-motor-literature/)) ++ The Manubot 2018 Development Proposal ([source](https://github.com/greenelab/manufund-2018), [manuscript](https://greenelab.github.io/manufund-2018/)) If you are using the Manubot, feel free to submit a pull request to add your manuscript to the list above. diff --git a/build/README.md b/build/README.md index 1deb31e..19b3723 100644 --- a/build/README.md +++ b/build/README.md @@ -12,11 +12,15 @@ Please reference [Pull Request #40](https://github.com/greenelab/manubot-rootsto ## Environment -Install the [conda](https://conda.io) environment specified in [`environment.yml`](environment.yml) by running: +Install or update the [conda](https://conda.io) environment specified in [`environment.yml`](environment.yml) by running: ```sh +# If the manubot environment already exists, remove it first +conda env remove --name manubot + +# Install the environment conda env create --file environment.yml ``` -Activate with `source activate manubot`. +Activate with `conda activate manubot` (assumes `conda` version of [at least](https://github.com/conda/conda/blob/9d759d8edeb86569c25f6eb82053f09581013a2a/CHANGELOG.md#440-2017-12-20) 4.4). The environment should successfully install on both Linux and macOS (and possibly Windows). diff --git a/build/environment.yml b/build/environment.yml index a8ca1b4..30eff14 100644 --- a/build/environment.yml +++ b/build/environment.yml @@ -4,24 +4,24 @@ channels: dependencies: - conda-forge::cairo=1.14.10 - conda-forge::cffi=1.11.2 + - conda-forge::ghp-import=0.5.5 - conda-forge::jinja2=2.10 - - conda-forge::pandas=0.21.0 - - conda-forge::pandoc=2.0.4 - - conda-forge::python=3.6.3 + - conda-forge::pandas=0.22.0 + - conda-forge::pandoc=2.1 + - conda-forge::python=3.6.4 - conda-forge::pyyaml=3.12 - conda-forge::requests=2.18.4 - conda-forge::watchdog=0.8.3 - pip: - errorhandler==2.0.1 - - ghp-import==0.5.5 - git+https://github.com/greenelab/manubot@33ff36728e23f3c685f3b00a6a26f19a6fcbf68e - - opentimestamps-client==0.5.0 - - opentimestamps==0.1.0 - - pandoc-eqnos==0.18.1 - - pandoc-fignos==0.21.1 - - pandoc-tablenos==0.17.1 + - opentimestamps-client==0.5.1 + - opentimestamps==0.2.0.1 + - pandoc-eqnos==1.0.0 + - pandoc-fignos==1.0.0 + - pandoc-tablenos==1.0.0 - pybase62==0.3.2 - pysha3==1.0.2 - - python-bitcoinlib==0.8.0 + - python-bitcoinlib==0.9.0 - requests-cache==0.4.13 - weasyprint==0.41 diff --git a/content/02.delete-me.md b/content/02.delete-me.md index 1bd7a85..ba79516 100644 --- a/content/02.delete-me.md +++ b/content/02.delete-me.md @@ -9,7 +9,7 @@ See [`USAGE.md`](https://github.com/greenelab/manubot-rootstock/blob/master/USAG The Manubot project began with the [Deep Review](https://github.com/greenelab/deep-review), where it was used to compose a highly-collaborative review article [@doi:10.1101/142760]. Other manuscripts that were created with Manubot include: -+ The Sci-Hub Coverage Study ([GitHub](https://github.com/greenelab/scihub-manuscript), [HTML manuscript](https://greenelab.github.io/scihub-manuscript/)) [@doi:10.7287/peerj.preprints.3100] ++ The Sci-Hub Coverage Study ([GitHub](https://github.com/greenelab/scihub-manuscript), [HTML manuscript](https://greenelab.github.io/scihub-manuscript/)) + Michael Zietz's Report for the Vagelos Scholars Program ([GitHub](https://github.com/zietzm/Vagelos2017), [HTML manuscript](https://zietzm.github.io/Vagelos2017/)) [@doi:10.6084/m9.figshare.5346577] If you notice a problem with Manubot, it's best to submit an upstream fix to the appropriate repository: From b140c766ef4e2bbbfc506f6298d80502c85ecdce Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Thu, 11 Jan 2018 13:19:28 -0500 Subject: [PATCH 169/216] Enable PubMed ID citations (#98) Upgrade Manubot to https://github.com/greenelab/manubot/commit/b53f487026954729ce80ca963f3838ebca85a144 Refs https://github.com/greenelab/manubot/pull/24 --- USAGE.md | 9 ++++++--- build/environment.yml | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/USAGE.md b/USAGE.md index 9b1b379..f5e3fcd 100644 --- a/USAGE.md +++ b/USAGE.md @@ -76,9 +76,10 @@ Therefore, citations must be of the following form: `@source:identifier`, where When choosing which source to use for a citation, we recommend the following order: 1. DOI (Digital Object Identifier), cite like `@doi:10.15363/thinklab.4`. -2. PubMed ID, cite like `@pmid:26158728`. -3. _arXiv_ ID, cite like `@arxiv:1508.06576v2`. -4. URL / webpage, cite like `@url:http://openreview.net/pdf?id=Sk-oDY9ge`. +2. PubMed Central ID, cite like `@pmcid:PMC4497619`. +3. PubMed ID, cite like `@pmid:26158728`. +4. _arXiv_ ID, cite like `@arxiv:1508.06576v2`. +5. URL / webpage, cite like `@url:http://openreview.net/pdf?id=Sk-oDY9ge`. Cite multiple items at once like: @@ -87,6 +88,8 @@ Here is a sentence with several citations [@doi:10.15363/thinklab.4; @pmid:26158 ``` Note that multiple citations must be semicolon separated. +Be careful not to cite the same study using identifiers from multiple sources. +For example, the following citations all refer to the same study, but will be treated as separate references: `[@doi:10.7717/peerj.705; @pmcid:PMC4304851; @pmid:25648772]`. #### Citation tags diff --git a/build/environment.yml b/build/environment.yml index 30eff14..856736a 100644 --- a/build/environment.yml +++ b/build/environment.yml @@ -14,7 +14,7 @@ dependencies: - conda-forge::watchdog=0.8.3 - pip: - errorhandler==2.0.1 - - git+https://github.com/greenelab/manubot@33ff36728e23f3c685f3b00a6a26f19a6fcbf68e + - git+https://github.com/greenelab/manubot@b53f487026954729ce80ca963f3838ebca85a144 - opentimestamps-client==0.5.1 - opentimestamps==0.2.0.1 - pandoc-eqnos==1.0.0 From a673a3e6a2122fc01c9ca19fbb26b729895874aa Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Sun, 14 Jan 2018 14:39:19 -0500 Subject: [PATCH 170/216] environment: explicitly set pandoc-xnos version (#99) Pinning pandoc-eqnos, pandoc-fignos, and pandoc-tablenos, but allowing them to trigger the installation of the latest pandoc-xnos version (which is a dependency) caused issues. See https://github.com/greenelab/deep-review/pull/774#issuecomment-357520693 --- build/environment.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/build/environment.yml b/build/environment.yml index 856736a..e5834be 100644 --- a/build/environment.yml +++ b/build/environment.yml @@ -17,9 +17,10 @@ dependencies: - git+https://github.com/greenelab/manubot@b53f487026954729ce80ca963f3838ebca85a144 - opentimestamps-client==0.5.1 - opentimestamps==0.2.0.1 - - pandoc-eqnos==1.0.0 - - pandoc-fignos==1.0.0 - - pandoc-tablenos==1.0.0 + - pandoc-eqnos==1.2.0 + - pandoc-fignos==1.2.0 + - pandoc-tablenos==1.2.0 + - pandoc-xnos==0.15 - pybase62==0.3.2 - pysha3==1.0.2 - python-bitcoinlib==0.9.0 From 744a8ca69ffb18f0ed68da6424886458ee5e5497 Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Sun, 14 Jan 2018 16:04:54 -0500 Subject: [PATCH 171/216] Environment updates on 2018-01-14 (#100) Upgrade Manubot with citations.tsv error message improvements Upgrade to weasyprint v0.42 http://weasyprint.readthedocs.io/en/latest/changelog.html#version-0-42 --- build/environment.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/environment.yml b/build/environment.yml index e5834be..ac8bdf4 100644 --- a/build/environment.yml +++ b/build/environment.yml @@ -14,7 +14,7 @@ dependencies: - conda-forge::watchdog=0.8.3 - pip: - errorhandler==2.0.1 - - git+https://github.com/greenelab/manubot@b53f487026954729ce80ca963f3838ebca85a144 + - git+https://github.com/greenelab/manubot@7ca386db0610d1e1113abba10c109dcffd3e4c9a - opentimestamps-client==0.5.1 - opentimestamps==0.2.0.1 - pandoc-eqnos==1.2.0 @@ -25,4 +25,4 @@ dependencies: - pysha3==1.0.2 - python-bitcoinlib==0.9.0 - requests-cache==0.4.13 - - weasyprint==0.41 + - weasyprint==0.42 From d946d9c64223a232ce29301c509924200dd364e2 Mon Sep 17 00:00:00 2001 From: Anthony Gitter Date: Tue, 16 Jan 2018 14:18:16 -0600 Subject: [PATCH 172/216] Update usage instructions for citation tags --- USAGE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/USAGE.md b/USAGE.md index f5e3fcd..cf55ddd 100644 --- a/USAGE.md +++ b/USAGE.md @@ -95,7 +95,7 @@ For example, the following citations all refer to the same study, but will be tr The system also supports citation tags, which are recommended for the following applications: -1. A citation's identifier contains forbidden characters, such as `;` or ending with a non-alphanumeric character other than `/`. +1. A citation's identifier contains forbidden characters, such as `;` or `=`, or ends with a non-alphanumeric character other than `/`. In these instances, you must use a tag. 2. A single reference is cited many times. Therefore, it might make sense to define a tag, so if the citation updates (e.g. a newer version becomes available), only a single change is required. From 0156d68cc69327097f506145324b008268d335cf Mon Sep 17 00:00:00 2001 From: Anthony Gitter Date: Fri, 19 Jan 2018 11:29:25 -0600 Subject: [PATCH 173/216] Suggest issues for general inquiries --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 7d17415..7853d61 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,8 @@ Manubot automates citations and references, versions manuscripts using git, and The [Manubot Rootstock repository](https://git.io/vQSvo) is a general purpose template for creating new Manubot instances, as detailed in [`SETUP.md`](SETUP.md). See [`USAGE.md`](USAGE.md) for documentation how to write a manuscript. +Please open [an issue](https://github.com/greenelab/manubot-rootstock/issues) for questions related to Manubot usage, bug reports, or general inquiries. + ### Repository directories & files The directories are as follows: From a535f7b872df775b1557f90b01402bbd56d17517 Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Thu, 1 Feb 2018 06:55:51 -0500 Subject: [PATCH 174/216] Webpage: specify letter page size in CSS (#107) Closes https://github.com/greenelab/manubot-rootstock/issues/105 --- webpage/github-pandoc.css | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/webpage/github-pandoc.css b/webpage/github-pandoc.css index 6445e2b..3189ab3 100644 --- a/webpage/github-pandoc.css +++ b/webpage/github-pandoc.css @@ -87,8 +87,9 @@ body { margin: 0; } -/***Page margins when printing***/ -@page { +/***Page size and margins when printing***/ +@page { + size: Letter; margin-top: 19mm; margin-bottom: 16mm; margin-left: 0mm; From 310dd078c4c60bedd63442f93906f26a4ea25a4b Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Tue, 6 Feb 2018 15:35:47 -0500 Subject: [PATCH 175/216] Update environment on 2018-02-06 (#108) Fix upstream issues: Extra brackets around citations in figure captions https://github.com/jgm/pandoc/issues/4272 WeasyPrint 0.42 gets stuck https://github.com/Kozea/WeasyPrint/issues/560 Updated pandoc-xnos with better semantic versioning https://github.com/tomduck/pandoc-fignos/issues/46 --- build/environment.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/build/environment.yml b/build/environment.yml index ac8bdf4..071bd66 100644 --- a/build/environment.yml +++ b/build/environment.yml @@ -7,22 +7,22 @@ dependencies: - conda-forge::ghp-import=0.5.5 - conda-forge::jinja2=2.10 - conda-forge::pandas=0.22.0 - - conda-forge::pandoc=2.1 + - conda-forge::pandoc=2.1.1 - conda-forge::python=3.6.4 - conda-forge::pyyaml=3.12 - conda-forge::requests=2.18.4 - conda-forge::watchdog=0.8.3 - pip: - errorhandler==2.0.1 - - git+https://github.com/greenelab/manubot@7ca386db0610d1e1113abba10c109dcffd3e4c9a + - git+https://github.com/greenelab/manubot@27456594c407f3fed35804db4bcf3b5bba88a716 - opentimestamps-client==0.5.1 - opentimestamps==0.2.0.1 - pandoc-eqnos==1.2.0 - pandoc-fignos==1.2.0 - pandoc-tablenos==1.2.0 - - pandoc-xnos==0.15 + - pandoc-xnos==1.0.1 - pybase62==0.3.2 - pysha3==1.0.2 - python-bitcoinlib==0.9.0 - requests-cache==0.4.13 - - weasyprint==0.42 + - weasyprint==0.42.2 From cc3b5085dd6a9851b8dbe93221a3878d3cc5cee3 Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Thu, 15 Feb 2018 13:39:07 -0500 Subject: [PATCH 176/216] Fix name with space issue (#110) git's user.name config was being set to first names only, since names are space separated. Add quotes to prevent this. --- ci/deploy.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/deploy.sh b/ci/deploy.sh index cf764ce..3cd61ec 100644 --- a/ci/deploy.sh +++ b/ci/deploy.sh @@ -15,7 +15,7 @@ ots stamp \ # Configure git git config --global push.default simple git config --global user.email `git log --max-count=1 --format='%ae'` -git config --global user.name `git log --max-count=1 --format='%an'` +git config --global user.name "`git log --max-count=1 --format='%an'`" git checkout $TRAVIS_BRANCH git remote set-url origin git@github.com:$TRAVIS_REPO_SLUG.git From 29cd993b0b6b9c41273e306f4c17262ffdbdf546 Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Fri, 16 Feb 2018 13:49:56 -0500 Subject: [PATCH 177/216] Versioned webpage using commit directories (#101) Closes https://github.com/greenelab/manubot-rootstock/issues/96 --- .gitignore | 2 +- README.md | 6 ++ SETUP.md | 8 +- build/assets/redirect-template.html | 19 ++++ build/webpage.py | 155 ++++++++++++++++++++++++++++ ci/deploy.sh | 15 ++- content/00.front-matter.md | 3 + webpage/README.md | 21 +++- webpage/images | 2 +- webpage/index.html | 2 +- webpage/manuscript.pdf | 2 +- 11 files changed, 222 insertions(+), 13 deletions(-) create mode 100644 build/assets/redirect-template.html create mode 100644 build/webpage.py diff --git a/.gitignore b/.gitignore index d5a66c7..6befc7c 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,7 @@ output/* !output/README.md -webpage/*.ots +webpage/v # Manubot cache directory ci/cache diff --git a/README.md b/README.md index 7853d61..3fe66dd 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,12 @@ sh build/build.sh # when a change is detected. sh build/autobuild.sh +# At this point, the HTML & PDF outputs will have been created. The remaining +# commands are for serving the webpage to view the HTML manuscript locally. + +# Configure the webpage directory +python build/webpage.py + # View the manuscript locally at http://localhost:8000/ cd webpage python -m http.server diff --git a/SETUP.md b/SETUP.md index dcb1403..9943282 100644 --- a/SETUP.md +++ b/SETUP.md @@ -32,8 +32,12 @@ cd $REPO # Configure remotes and branches git remote add rootstock https://github.com/greenelab/manubot-rootstock.git -git checkout gh-pages -git checkout output +git checkout --orphan gh-pages +git rm -r --force . +git commit --allow-empty \ + --message "Initialize empty branch" \ + --message "[ci skip]" +git checkout -b output git checkout master # Option A: Set origin URL using its web address diff --git a/build/assets/redirect-template.html b/build/assets/redirect-template.html new file mode 100644 index 0000000..9fd83c2 --- /dev/null +++ b/build/assets/redirect-template.html @@ -0,0 +1,19 @@ + + + + + + + + Page Redirection + + + If you are not redirected automatically, follow this link. + + diff --git a/build/webpage.py b/build/webpage.py new file mode 100644 index 0000000..bf0b021 --- /dev/null +++ b/build/webpage.py @@ -0,0 +1,155 @@ +import argparse +import os +import pathlib +import shutil +import subprocess + + +def parse_arguments(): + """ + Read and process command line arguments. + """ + parser = argparse.ArgumentParser() + parser.add_argument( + '--checkout', + nargs='?', const='gh-pages', default=None, + help='branch to checkout /v directory contents from. For example, --checkout=upstream/gh-pages. --checkout is equivalent to --checkout=gh-pages. If --checkout is ommitted, no checkout is performed.', + ) + parser.add_argument( + '--version', + default=os.environ.get('TRAVIS_COMMIT', 'local'), + help="Used to create webpage/v/{version} directory. Generally a commit hash, tag, or 'local'", + ) + args = parser.parse_args() + return args + + +def configure_directories(args): + """ + Add directories to args and create them if neccessary. + Note that versions_directory is the parent of version_directory. + """ + args_dict = vars(args) + + # Directory where Manubot outputs reside + args_dict['output_directory'] = pathlib.Path('output') + + # Set webpage directory + args_dict['webpage_directory'] = pathlib.Path('webpage') + + # Create webpage/v directory (if it doesn't already exist) + args_dict['versions_directory'] = args.webpage_directory.joinpath('v') + args.versions_directory.mkdir(exist_ok=True) + + # Checkout existing version directories + checkout_existing_versions(args) + + # Create empty webpage/v/version directory + version_directory = args.versions_directory.joinpath(args.version) + if version_directory.is_dir(): + print(f'{version_directory} exists: replacing it with an empty directory') + shutil.rmtree(version_directory) + version_directory.mkdir() + args_dict['version_directory'] = version_directory + + # Symlink webpage/v/latest to point to webpage/v/commit + latest_directory = args.versions_directory.joinpath('latest') + if latest_directory.is_symlink() or latest_directory.is_file(): + latest_directory.unlink() + elif latest_directory.is_dir(): + shutil.rmtree(latest_directory) + latest_directory.symlink_to(args.version, target_is_directory=True) + args_dict['latest_directory'] = latest_directory + + # Create freeze directory + freeze_directory = args.versions_directory.joinpath('freeze') + freeze_directory.mkdir(exist_ok=True) + args_dict['freeze_directory'] = freeze_directory + + return args + + +def checkout_existing_versions(args): + """ + Must populate webpage/v from the gh-pages branch to get history + + References: + http://clubmate.fi/git-checkout-file-or-directories-from-another-branch/ + https://stackoverflow.com/a/2668947/4651668 + https://stackoverflow.com/a/16493707/4651668 + + Command modeled after: + git --work-tree=webpage checkout upstream/gh-pages -- v + """ + if not args.checkout: + return + command = [ + 'git', + f'--work-tree={args.webpage_directory}', + 'checkout', + args.checkout, + '--', + 'v', + ] + print('Attempting checkout with the following command:', ' '.join(command), sep='\n') + process = subprocess.run(command, stderr=subprocess.PIPE) + if process.returncode == 0: + # Addresses an odd behavior where git checkout stages v/* files that don't actually exist + subprocess.run(['git', 'add', 'v']) + else: + print(f'Checkout returned a nonzero exit status. See stderr:\n{process.stderr.decode()}') + + +def create_version(args): + """ + Populate the version directory for a new version. + """ + + # Copy content/images to webpage/v/commit/images + shutil.copytree( + src=pathlib.Path('content/images'), + dst=args.version_directory.joinpath('images'), + ) + + # Copy output files to to webpage/v/version/ + renamer = { + 'manuscript.html': 'index.html', + 'manuscript.pdf': 'manuscript.pdf', + } + for src, dst in renamer.items(): + shutil.copy2( + src=args.output_directory.joinpath(src), + dst=args.version_directory.joinpath(dst), + ) + + # Copy webpage/github-pandoc.css to to webpage/v/version/github-pandoc.css + shutil.copy2( + src=args.webpage_directory.joinpath('github-pandoc.css'), + dst=args.version_directory.joinpath('github-pandoc.css'), + ) + + # Create v/freeze to redirect to v/commit + path = pathlib.Path('build/assets/redirect-template.html') + redirect_html = path.read_text() + redirect_html = redirect_html.format(url=f'../{args.version}/') + args.freeze_directory.joinpath('index.html').write_text(redirect_html) + + +def get_versions(): + """ + Extract versions from the webpage/v directory, which should each contain + a manuscript. + """ + versions = {x.name for x in args.versions_directory.iterdir() if x.is_dir()} + versions -= {'freeze', 'latest'} + versions = sorted(versions) + return versions + + +if __name__ == '__main__': + args = parse_arguments() + configure_directories(args) + print(args) + create_version(args) + versions = get_versions() + print(versions) diff --git a/ci/deploy.sh b/ci/deploy.sh index 3cd61ec..96e40f0 100644 --- a/ci/deploy.sh +++ b/ci/deploy.sh @@ -7,11 +7,6 @@ export REPO_NAME=`basename $TRAVIS_REPO_SLUG` envsubst < webpage/README.md > webpage/README-complete.md mv webpage/README-complete.md webpage/README.md -# Generate OpenTimestamps -ots stamp \ - webpage/index.html \ - webpage/manuscript.pdf - # Configure git git config --global push.default simple git config --global user.email `git log --max-count=1 --format='%ae'` @@ -34,6 +29,16 @@ ssh-add ci/deploy.key git remote set-branches --add origin gh-pages output git fetch origin gh-pages:gh-pages output:output +# Configure versioned webpage +python build/webpage.py \ + --checkout=gh-pages \ + --version=$TRAVIS_COMMIT + +# Generate OpenTimestamps +ots stamp \ + webpage/v/$TRAVIS_COMMIT/index.html \ + webpage/v/$TRAVIS_COMMIT/manuscript.pdf + # Commit message MESSAGE="\ `git log --max-count=1 --format='%s'` diff --git a/content/00.front-matter.md b/content/00.front-matter.md index 22dc935..3a34cd9 100644 --- a/content/00.front-matter.md +++ b/content/00.front-matter.md @@ -15,6 +15,9 @@ This manuscript was automatically generated from [{{ci_source.repo_slug}}@{{ci_source.commit | truncate(length=7, end='', leeway=0)}}](https://github.com/{{ci_source.repo_slug}}/tree/{{ci_source.commit}}) {% endif -%} on {{date}}. +{% if ci_source is defined -%} +The permalink for this manuscript version is . +{% endif -%} ## Authors diff --git a/webpage/README.md b/webpage/README.md index 407f3e9..b347ecc 100644 --- a/webpage/README.md +++ b/webpage/README.md @@ -1,6 +1,8 @@ # Output directory containing the formatted manuscript The [`gh-pages`](https://github.com/$TRAVIS_REPO_SLUG/tree/gh-pages) branch hosts the contents of this directory at https://$OWNER_NAME.github.io/$REPO_NAME/. +The permalink for this webpage version is https://$OWNER_NAME.github.io/$REPO_NAME/v/$TRAVIS_COMMIT/. +To redirect to the permalink for the latest manuscript version at anytime, use the link https://$OWNER_NAME.github.io/$REPO_NAME/v/freeze/. ## Files @@ -9,8 +11,23 @@ This directory contains the following files, which are mostly ignored on the `ma + [`index.html`](index.html) is an HTML manuscript. + [`github-pandoc.css`](github-pandoc.css) sets the display style for `index.html`. + [`manuscript.pdf`](manuscript.pdf) is a PDF manuscript. -+ `*.ots` files are OpenTimestamps which can be used to verify manuscript existence at or before a given time. - [OpenTimestamps](opentimestamps.org) uses the Bitcoin blockchain to attest to file hash existence. + +The `v` directory contains directories for each manuscript version. +In general, a version is identified by the commit hash of the source content that created it. + +### Timestamps + +The `*.ots` files in version directories are OpenTimestamps which can be used to verify manuscript existence at or before a given time. +[OpenTimestamps](https://opentimestamps.org/) uses the Bitcoin blockchain to attest to file hash existence. +The `deploy.sh` script run during continuous deployment creates the `.ots` files. +However, these files are initially dependent on calendar services and must be upgraded at a later time by running the following in the `gh-pages` branch: + +```sh +# Requires a local bitcoin node with JSON-RPC configured +ots upgrade v/*/*.ots +rm v/*/*.ots.bak +git add v/*/*.ots +``` ## Source diff --git a/webpage/images b/webpage/images index a0fc219..2b55bac 120000 --- a/webpage/images +++ b/webpage/images @@ -1 +1 @@ -../content/images \ No newline at end of file +v/latest/images \ No newline at end of file diff --git a/webpage/index.html b/webpage/index.html index 936379e..2f982ac 120000 --- a/webpage/index.html +++ b/webpage/index.html @@ -1 +1 @@ -../output/manuscript.html \ No newline at end of file +v/latest/index.html \ No newline at end of file diff --git a/webpage/manuscript.pdf b/webpage/manuscript.pdf index 90c08c6..b886727 120000 --- a/webpage/manuscript.pdf +++ b/webpage/manuscript.pdf @@ -1 +1 @@ -../output/manuscript.pdf \ No newline at end of file +v/latest/manuscript.pdf \ No newline at end of file From b7a9de896fa6e056a6adf983592166232349cf34 Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Sat, 17 Feb 2018 12:51:13 -0500 Subject: [PATCH 178/216] autobuild.sh: include webpage.py execution (#113) Most autobuild users will want to automatically update the webpage. See discussion at https://github.com/greenelab/manubot-rootstock/pull/101#issuecomment-366405495 Update README to better document autobuild. --- README.md | 13 ++++++++----- build/autobuild.sh | 4 ++-- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 3fe66dd..7b55c40 100644 --- a/README.md +++ b/README.md @@ -46,13 +46,9 @@ Then, you can build the manuscript on POSIX systems by running the following com # Activate the manubot conda environment (assumes conda version >= 4.4) conda activate manubot -# Build the manuscript +# Build the manuscript, saving outputs to the output directory sh build/build.sh -# Or monitor the content directory, and automatically rebuild the manuscript -# when a change is detected. -sh build/autobuild.sh - # At this point, the HTML & PDF outputs will have been created. The remaining # commands are for serving the webpage to view the HTML manuscript locally. @@ -64,6 +60,13 @@ cd webpage python -m http.server ``` +Sometimes it's helpful to monitor the content directory and automatically rebuild the manuscript when a change is detected. +The following command, while running, will trigger both the `build.sh` and `webpage.py` scripts upon content changes: + +```sh +sh build/autobuild.sh +``` + ### Continuous Integration [![Build Status](https://travis-ci.org/greenelab/manubot-rootstock.svg?branch=master)](https://travis-ci.org/greenelab/manubot-rootstock) diff --git a/build/autobuild.sh b/build/autobuild.sh index 404e3d2..d6a5313 100644 --- a/build/autobuild.sh +++ b/build/autobuild.sh @@ -1,6 +1,6 @@ -# Automatically rebuild manuscript when content changes +# Automatically rebuild mansucript outputs and the webpage when content changes # Depends on watchdog https://github.com/gorakhargosh/watchdog watchmedo shell-command \ --wait \ - --command='sh build/build.sh' \ + --command='sh build/build.sh && python build/webpage.py' \ content From 9438011c1a17a166e5c4e6b8b1dd3391f83fe5c5 Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Mon, 26 Feb 2018 17:35:12 -0500 Subject: [PATCH 179/216] Upgrade miniconda on Travis to 4.4.10 Merges https://github.com/greenelab/manubot-rootstock/pull/115 The method for enabling conda has changed. See notes at https://github.com/conda/conda/blob/53954c0d6785510587cd755a0cee4e177dca8455/CHANGELOG.md#recommended-change-to-enable-conda-in-your-shell --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2267dab..5592ad1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,16 +5,16 @@ branches: only: - master before_install: - - wget https://repo.continuum.io/miniconda/Miniconda3-4.3.31-Linux-x86_64.sh + - wget https://repo.continuum.io/miniconda/Miniconda3-4.4.10-Linux-x86_64.sh --output-document miniconda.sh - bash miniconda.sh -b -p $HOME/miniconda - - export PATH="$HOME/miniconda/bin:$PATH" + - source $HOME/miniconda/etc/profile.d/conda.sh - hash -r - conda config --set always_yes yes --set changeps1 no - conda info --all install: - conda env create --quiet --file build/environment.yml - - source activate manubot + - conda activate manubot script: - sh build/build.sh cache: From fffcaf0c65ad836d54abbea5361906da60ff2db0 Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Mon, 26 Feb 2018 17:36:22 -0500 Subject: [PATCH 180/216] front matter: hyperlink permalink for brevity Merges https://github.com/greenelab/manubot-rootstock/pull/114 --- content/00.front-matter.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/content/00.front-matter.md b/content/00.front-matter.md index 3a34cd9..c438b3d 100644 --- a/content/00.front-matter.md +++ b/content/00.front-matter.md @@ -10,14 +10,15 @@ _A DOI-citable version of this manuscript is available at -This manuscript was automatically generated +This manuscript {% if ci_source is defined -%} -from [{{ci_source.repo_slug}}@{{ci_source.commit | truncate(length=7, end='', leeway=0)}}](https://github.com/{{ci_source.repo_slug}}/tree/{{ci_source.commit}}) +([permalink](https://{{ci_source.repo_owner}}.github.io/{{ci_source.repo_name}}/v/{{ci_source.commit}}/)) {% endif -%} -on {{date}}. +was automatically generated {% if ci_source is defined -%} -The permalink for this manuscript version is . +from [{{ci_source.repo_slug}}@{{ci_source.commit | truncate(length=7, end='', leeway=0)}}](https://github.com/{{ci_source.repo_slug}}/tree/{{ci_source.commit}}) {% endif -%} +on {{date}}. ## Authors From d91a7bda64733d5f22982ae54d07f4829ba5f87e Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Wed, 28 Mar 2018 22:13:30 -0400 Subject: [PATCH 181/216] CSL: do not collapse numeric citations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Change style of citations from [1–4] to [1,2,3,4] Merges https://github.com/greenelab/manubot-rootstock/pull/121 --- build/assets/style.csl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/assets/style.csl b/build/assets/style.csl index a7dd0d8..bdbffe2 100644 --- a/build/assets/style.csl +++ b/build/assets/style.csl @@ -34,7 +34,7 @@ - + From f4cc39a481f8811a53832ed00272f69d4e269615 Mon Sep 17 00:00:00 2001 From: Ogun Adebali Date: Sat, 31 Mar 2018 07:13:22 -0400 Subject: [PATCH 182/216] travis login information added (https://github.com/greenelab/manubot-rootstock/pull/119) * travis login information added * single quote and inline sentence * Add newline --- SETUP.md | 1 + 1 file changed, 1 insertion(+) diff --git a/SETUP.md b/SETUP.md index 9943282..554cabd 100644 --- a/SETUP.md +++ b/SETUP.md @@ -87,6 +87,7 @@ Give the key a descriptive title, such as "Travis CI Manubot". For the next step, you need the [Travis command line client](https://github.com/travis-ci/travis.rb) installed. This program is a Ruby gem: install it with `gem install travis` (not `apt install travis`, which is a different program). +After the install, you will need to provide your credentials to login to travis with `travis login --org`. ```sh travis encrypt-file \ From 81945d9f23384108e8bc5c25e885d929668e12d8 Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Thu, 5 Apr 2018 16:13:07 -0400 Subject: [PATCH 183/216] build.sh: remove long args incompatible with macOS Merges https://github.com/greenelab/manubot-rootstock/pull/123 Refs https://github.com/greenelab/manubot-rootstock/issues/120 --- build/build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/build.sh b/build/build.sh index e7c1b1e..7a79b26 100644 --- a/build/build.sh +++ b/build/build.sh @@ -44,7 +44,7 @@ pandoc --verbose \ # Create PDF output echo "Exporting PDF manuscript" -ln --symbolic content/images images +ln -s content/images images pandoc \ --from=markdown \ --to=html5 \ @@ -60,7 +60,7 @@ pandoc \ --css=webpage/github-pandoc.css \ --output=output/manuscript.pdf \ $INPUT_PATH -rm --recursive images +rm -r images # Create DOCX output when user specifies to do so if [ "$BUILD_DOCX" = "true" ]; From 908848162ef68f0606d12880d381f563faf01947 Mon Sep 17 00:00:00 2001 From: Anthony Gitter Date: Mon, 18 Jun 2018 10:06:37 -0500 Subject: [PATCH 184/216] build/README.md: minor fix to build instructions Merges https://github.com/greenelab/manubot-rootstock/pull/125 --- build/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/README.md b/build/README.md index 19b3723..98f1981 100644 --- a/build/README.md +++ b/build/README.md @@ -1,7 +1,7 @@ # Building the manuscript [`build.sh`](build.sh) builds the repository. -`sh build.sh` should be executed from the root directory of the repository. +`sh build/build.sh` should be executed from the root directory of the repository. To build a DOCX file of the manuscript, set the `BUILD_DOCX` environment variable to `true`. For example, use the command `BUILD_DOCX=true sh build/build.sh`. From 9aec6f5c2eeca9002a5ce4dc06808fc2fbec24f3 Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Tue, 10 Jul 2018 11:54:40 -0400 Subject: [PATCH 185/216] Add 3 additional example manuscripts to USAGE.md Merges https://github.com/greenelab/manubot-rootstock/pull/127 Add TFSEE, global emissions, & yt 3.0 manuscripts --- USAGE.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/USAGE.md b/USAGE.md index cf55ddd..df02493 100644 --- a/USAGE.md +++ b/USAGE.md @@ -152,7 +152,7 @@ If you experience any issues with the Manubot or would like to contribute to its ## Examples -For additional examples, check out existing manuscripts that use the Manubot: +For additional examples, check out existing manuscripts that use the Manubot (some of which are still in progress): + Satoshi Nakamoto's Bitcoin Whitepaper ([source](https://github.com/dhimmel/bitcoin-whitepaper/), [manuscript](https://dhimmel.github.io/bitcoin-whitepaper/), [commentary](https://steemit.com/manubot/@dhimmel/how-i-used-the-manubot-to-reproduce-the-bitcoin-whitepaper)) + The Sci-Hub Coverage Study ([source](https://github.com/greenelab/scihub-manuscript), [manuscript](https://greenelab.github.io/scihub-manuscript/)) @@ -161,6 +161,9 @@ For additional examples, check out existing manuscripts that use the Manubot: + The Meta Review ([source](https://github.com/greenelab/meta-review), [manuscript](https://greenelab.github.io/meta-review/)) + The Project Rephetio Manuscript ([source](https://github.com/dhimmel/rephetio-manuscript/), [manuscript](https://dhimmel.github.io/rephetio-manuscript/)) + A Literature Review for Project Planning by David Slochower ([source](https://github.com/slochower/synthetic-motor-literature), [manuscript](https://slochower.github.io/synthetic-motor-literature/)) ++ The TFSEE Manuscript by Venkat Malladi et al. ([source](https://github.com/vsmalladi/tfsee-manuscript), [manuscript](https://vsmalladi.github.io/tfsee-manuscript/)) ++ Creating a Global Emissions Timeseries dataset by Robert Gieseke et al. ([source](https://github.com/openclimatedata/global-emissions), [manuscript](https://openclimatedata.github.io/global-emissions/)) ++ The yt 3.0 methods paper ([source](https://github.com/yt-project/yt-3.0-paper), [manuscript](https://yt-project.github.io/yt-3.0-paper/)) + The Manubot 2018 Development Proposal ([source](https://github.com/greenelab/manufund-2018), [manuscript](https://greenelab.github.io/manufund-2018/)) If you are using the Manubot, feel free to submit a pull request to add your manuscript to the list above. From 22b526073e0fe7a96cc35b612c47c371c92333df Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Fri, 3 Aug 2018 18:50:14 -0400 Subject: [PATCH 186/216] CSS: figure & table caption alignment Merges https://github.com/greenelab/manubot-rootstock/pull/131 Closes https://github.com/greenelab/manubot-rootstock/issues/130 * Left-align table caption text * figure img style: center img, left-align text --- webpage/github-pandoc.css | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/webpage/github-pandoc.css b/webpage/github-pandoc.css index 3189ab3..e0e5e81 100644 --- a/webpage/github-pandoc.css +++ b/webpage/github-pandoc.css @@ -13,8 +13,6 @@ article, aside, details, -figcaption, -figure, footer, header, hgroup, @@ -295,14 +293,25 @@ svg:not(:root) { Figures ========================================================================== */ -/** - * Address margin not present in IE 8/9 and Safari 5. - */ - figure { + display: block; + /* Address margin not present in IE 8/9 and Safari 5. */ margin: 0; } +figcaption { + caption-side: bottom; + display: block; + text-align: left; +} + +figure img { + display:block; + margin-left:auto; + margin-right:auto; + max-width:100%; +} + /* ========================================================================== Forms ========================================================================== */ @@ -447,6 +456,14 @@ textarea { Tables ========================================================================== */ +/** +* Specify table caption location and text alignment. +*/ +caption { + caption-side: top; + text-align: left; +} + /** * Remove most spacing between table cells. */ @@ -675,9 +692,6 @@ table tr { table tr:nth-child(2n) { background-color: #f8f8f8 } -img { - max-width: 100% -} code, tt { margin: 0 2px; From 3df864ecda8e22e738bf6f7f24833799b99cd961 Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Fri, 31 Aug 2018 09:42:31 -0700 Subject: [PATCH 187/216] Upgrade manubot & environment on 2018-08-22 Merges https://github.com/greenelab/manubot-rootstock/pull/132 * Upgrade manubot to use process subcommand * Upgrade environment to latest versions on 2018-08-22 * CSL style: block to display identifiers --- .travis.yml | 2 +- build/assets/style.csl | 12 +++++++++--- build/build.sh | 2 +- build/environment.yml | 32 ++++++++++++++++---------------- 4 files changed, 27 insertions(+), 21 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5592ad1..60cc09a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,7 @@ branches: only: - master before_install: - - wget https://repo.continuum.io/miniconda/Miniconda3-4.4.10-Linux-x86_64.sh + - wget https://repo.continuum.io/miniconda/Miniconda3-4.5.4-Linux-x86_64.sh --output-document miniconda.sh - bash miniconda.sh -b -p $HOME/miniconda - source $HOME/miniconda/etc/profile.d/conda.sh diff --git a/build/assets/style.csl b/build/assets/style.csl index bdbffe2..756c758 100644 --- a/build/assets/style.csl +++ b/build/assets/style.csl @@ -1,4 +1,4 @@ - + diff --git a/build/build.sh b/build/build.sh index 7a79b26..60e4adb 100644 --- a/build/build.sh +++ b/build/build.sh @@ -7,7 +7,7 @@ export LC_ALL=en_US.UTF-8 # Generate reference information echo "Retrieving and processing reference metadata" -manubot \ +manubot process \ --content-directory=content \ --output-directory=output \ --cache-directory=ci/cache \ diff --git a/build/environment.yml b/build/environment.yml index 071bd66..a77ba34 100644 --- a/build/environment.yml +++ b/build/environment.yml @@ -2,27 +2,27 @@ name: manubot channels: - conda-forge dependencies: - - conda-forge::cairo=1.14.10 - - conda-forge::cffi=1.11.2 + - conda-forge::cairo=1.14.12 + - conda-forge::cffi=1.11.5 - conda-forge::ghp-import=0.5.5 - conda-forge::jinja2=2.10 - - conda-forge::pandas=0.22.0 - - conda-forge::pandoc=2.1.1 - - conda-forge::python=3.6.4 + - conda-forge::pandas=0.23.4 + - conda-forge::pandoc=2.2.2 + - conda-forge::python=3.6.6 - conda-forge::pyyaml=3.12 - - conda-forge::requests=2.18.4 + - conda-forge::requests=2.19.1 - conda-forge::watchdog=0.8.3 - pip: - errorhandler==2.0.1 - - git+https://github.com/greenelab/manubot@27456594c407f3fed35804db4bcf3b5bba88a716 - - opentimestamps-client==0.5.1 - - opentimestamps==0.2.0.1 - - pandoc-eqnos==1.2.0 - - pandoc-fignos==1.2.0 - - pandoc-tablenos==1.2.0 - - pandoc-xnos==1.0.1 - - pybase62==0.3.2 + - git+https://github.com/greenelab/manubot@66a6efc6f4b84153a813aa423ec00725ed1417c5 + - opentimestamps-client==0.6.0 + - opentimestamps==0.4.0 + - pandoc-eqnos==1.3.0 + - pandoc-fignos==1.3.0 + - pandoc-tablenos==1.3.0 + - pandoc-xnos==1.1.1 + - pybase62==0.4.0 - pysha3==1.0.2 - - python-bitcoinlib==0.9.0 + - python-bitcoinlib==0.10.1 - requests-cache==0.4.13 - - weasyprint==0.42.2 + - weasyprint==0.42.3 From e83e51dcd89256403bb787c3d9a46e4ee8d04a9e Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Sat, 6 Oct 2018 10:01:27 -0400 Subject: [PATCH 188/216] CSL: display="block" every other group Merges https://github.com/greenelab/manubot-rootstock/pull/134 Previous blocks were causing a blank line after the author line before the journal-date-URL line in markdown output. Placing blocks at every other line seems to be the solution required for pandoc-citeproc / pandoc to properly place the newlines. --- build/assets/style.csl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/assets/style.csl b/build/assets/style.csl index 756c758..07ba262 100644 --- a/build/assets/style.csl +++ b/build/assets/style.csl @@ -51,7 +51,7 @@ - + @@ -60,7 +60,7 @@ - + From 830cdc088dee2a61452bd56d990e0277daa0a9af Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Mon, 8 Oct 2018 15:12:33 -0400 Subject: [PATCH 189/216] Update USAGE.md & environment on 2018-10-08 Merges https://github.com/greenelab/manubot-rootstock/pull/135 Closes https://github.com/greenelab/manubot/issues/59 * USAGE.md: suggested citation & acknowledgments Mostly copied from https://github.com/greenelab/manubot/blob/9d97ec347882bcd85ab6aee7a3b4734105ebfc15/README.md * Update environment on 2018-10-08 Updates to https://github.com/greenelab/manubot@9d97ec347882bcd85ab6aee7a3b4734105ebfc15 which is slightly past the manubot v0.2.0 release. jsonschema & jsonref should have been added to the environment previousely since they are required by manubot for JSON schema validation, but were not added. Do not upgrade to Python 3.7 due to collections DeprecationWarnings in several packages. Fixed warning in https://github.com/gazpachoking/jsonref/pull/26, but several remain. * Travis CI: update miniconda to 4.5.11 --- .travis.yml | 2 +- USAGE.md | 17 +++++++++++++++++ build/environment.yml | 10 ++++++---- 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 60cc09a..29420f4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,7 @@ branches: only: - master before_install: - - wget https://repo.continuum.io/miniconda/Miniconda3-4.5.4-Linux-x86_64.sh + - wget https://repo.continuum.io/miniconda/Miniconda3-4.5.11-Linux-x86_64.sh --output-document miniconda.sh - bash miniconda.sh -b -p $HOME/miniconda - source $HOME/miniconda/etc/profile.d/conda.sh diff --git a/USAGE.md b/USAGE.md index df02493..ddcdc07 100644 --- a/USAGE.md +++ b/USAGE.md @@ -167,3 +167,20 @@ For additional examples, check out existing manuscripts that use the Manubot (so + The Manubot 2018 Development Proposal ([source](https://github.com/greenelab/manufund-2018), [manuscript](https://greenelab.github.io/manufund-2018/)) If you are using the Manubot, feel free to submit a pull request to add your manuscript to the list above. + +## Citing Manubot + +To cite the Manubot project or for more information on its design and history, see `@url:https://greenelab.github.io/meta-review/`: + +> **Open collaborative writing with Manubot**
+Daniel S. Himmelstein, David R. Slochower, Venkat S. Malladi, Casey S. +Greene, Anthony Gitter
+_Manubot Preprint_ (2018) + +## Acknowledgments + +We would like to thank the contributors and funders whose support makes the Manubot project possible. +Specifically, Manubot development has been financially supported by: + +- the **Alfred P. Sloan Foundation** in [Grant G-2018-11163](https://sloan.org/grant-detail/8501) to [**@dhimmel**](https://github.com/dhimmel). +- the **Gordon & Betty Moore Foundation** ([**@DDD-Moore**](https://github.com/DDD-Moore)) in [Grant GBMF4552](https://www.moore.org/grant-detail?grantId=GBMF4552) to [**@cgreene**](https://github.com/cgreene). diff --git a/build/environment.yml b/build/environment.yml index a77ba34..cc38436 100644 --- a/build/environment.yml +++ b/build/environment.yml @@ -6,15 +6,17 @@ dependencies: - conda-forge::cffi=1.11.5 - conda-forge::ghp-import=0.5.5 - conda-forge::jinja2=2.10 + - conda-forge::jsonschema=2.6.0 - conda-forge::pandas=0.23.4 - - conda-forge::pandoc=2.2.2 - - conda-forge::python=3.6.6 - - conda-forge::pyyaml=3.12 + - conda-forge::pandoc=2.3.1 + - conda-forge::python=3.6.5 + - conda-forge::pyyaml=3.13 - conda-forge::requests=2.19.1 - conda-forge::watchdog=0.8.3 - pip: - errorhandler==2.0.1 - - git+https://github.com/greenelab/manubot@66a6efc6f4b84153a813aa423ec00725ed1417c5 + - git+https://github.com/greenelab/manubot@9d97ec347882bcd85ab6aee7a3b4734105ebfc15 + - jsonref==0.2 - opentimestamps-client==0.6.0 - opentimestamps==0.4.0 - pandoc-eqnos==1.3.0 From 6bb804c018d40f0ef4e7f79fcfa5600827f112ca Mon Sep 17 00:00:00 2001 From: Anthony Gitter Date: Tue, 9 Oct 2018 11:20:08 -0500 Subject: [PATCH 190/216] Case sensitive environment variables in setup instructions Merges https://github.com/greenelab/manubot-rootstock/pull/138 Closes https://github.com/greenelab/manubot-rootstock/issues/137 * Case sensitive environment variables * Update phrasing borrowing dhimmel ideas * Change account to username --- SETUP.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/SETUP.md b/SETUP.md index 554cabd..22a4f06 100644 --- a/SETUP.md +++ b/SETUP.md @@ -9,10 +9,12 @@ Setup is supported on Linux and macOS, but [**not on Windows**](https://github.c First, you must configure two environment variables (`OWNER` and `REPO`). These variables specify the GitHub repository for the manuscript (i.e. `https://github.com/OWNER/REPO`). +Make sure that the case of `OWNER` matches how your username is displayed on GitHub. +In general, assume that all commands in this setup are case-sensitive. **Edit the following commands with your manuscript's information:** ```sh -# GitHub account (change from greenelab) +# GitHub username (change from greenelab) OWNER=greenelab # Repository name (change from manubot-rootstock) REPO=manubot-rootstock From dcfe402544f90e0047a247cdd8d01a2bca08df54 Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Tue, 23 Oct 2018 15:23:20 -0400 Subject: [PATCH 191/216] Bugfix environment updates on 2018-10-23 Merges https://github.com/greenelab/manubot-rootstock/pull/140 Closes https://github.com/greenelab/manubot-rootstock/issues/136 Updates Manubot to fix empty date-parts issue Fix opentimestamps incompatible pinned versions --- build/environment.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build/environment.yml b/build/environment.yml index cc38436..73171db 100644 --- a/build/environment.yml +++ b/build/environment.yml @@ -15,16 +15,16 @@ dependencies: - conda-forge::watchdog=0.8.3 - pip: - errorhandler==2.0.1 - - git+https://github.com/greenelab/manubot@9d97ec347882bcd85ab6aee7a3b4734105ebfc15 + - git+https://github.com/greenelab/manubot@4e6a0f6d28220264c0d7892c732cb68c3e97c07a - jsonref==0.2 - opentimestamps-client==0.6.0 - - opentimestamps==0.4.0 + - opentimestamps==0.3.0 - pandoc-eqnos==1.3.0 - pandoc-fignos==1.3.0 - pandoc-tablenos==1.3.0 - pandoc-xnos==1.1.1 - pybase62==0.4.0 - pysha3==1.0.2 - - python-bitcoinlib==0.10.1 + - python-bitcoinlib==0.9.0 - requests-cache==0.4.13 - weasyprint==0.42.3 From 895924e6a8c693a0ee110ec410b153b7ceb09bc2 Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Tue, 30 Oct 2018 15:28:24 -0400 Subject: [PATCH 192/216] Enable raw citations & small USAGE updates Merges https://github.com/greenelab/manubot-rootstock/pull/141 * Update USAGE with current URLs & recs Use HTTPS URLs where possible Switch SVG embedding solution from rawgit.com, which is shutting down, to the native GitHub solution. https://twitter.com/dhimmel/status/1049361799244664834 * Enable raw citations Update USAGE.md with raw citation example. Also update citations in delete-me.md. --- USAGE.md | 44 +++++++++++++++++++++++++++-------------- build/environment.yml | 2 +- content/02.delete-me.md | 4 ++-- 3 files changed, 32 insertions(+), 18 deletions(-) diff --git a/USAGE.md b/USAGE.md index ddcdc07..5bd3507 100644 --- a/USAGE.md +++ b/USAGE.md @@ -1,18 +1,18 @@ # Manubot usage guidelines -This repository uses the [Manubot](https://github.com/greenelab/manubot) to automatically produce a manuscript from its source. +This repository uses [Manubot](https://github.com/greenelab/manubot-rootstock) to automatically produce a manuscript from the source in the [`content`](content) directory. ## Manubot markdown Manuscript text should be written in markdown files, which should be located in [`content`](content) with a digit prefix for ordering (e.g. `01.`, `02.`, etc.) and a `.md` extension. -For basic formatting, check out the [CommonMark Help](http://commonmark.org/help/) page for an introduction to the formatting options provided by standard markdown. -In addition, manubot supports an extended version of markdown, tailored for scholarly writing, which includes [Pandoc's Markdown](http://pandoc.org/MANUAL.html#pandocs-markdown) and the extensions discussed below. +For basic formatting, check out the [CommonMark Help](https://commonmark.org/help/) page for an introduction to the formatting options provided by standard markdown. +In addition, Manubot supports an extended version of markdown, tailored for scholarly writing, which includes [Pandoc's Markdown](https://pandoc.org/MANUAL.html#pandocs-markdown) and the extensions discussed below. Within a paragraph in markdown, single newlines are interpreted as whitespace (same as a space). A paragraph's source does not need to contain newlines. However, "one paragraph per line" makes the git diff less precise, leading to less granular review commenting, and makes conflicts more likely. -Therefore, we recommend using [semantic linefeeds](http://rhodesmill.org/brandon/2012/one-sentence-per-line/ "Semantic Linefeeds. Brandon Rhodes. 2012") — newlines between sentences. +Therefore, we recommend using [semantic linefeeds](https://rhodesmill.org/brandon/2012/one-sentence-per-line/ "Semantic Linefeeds. Brandon Rhodes. 2012") — newlines between sentences. We have found that "one sentence per line" is preferable to "word wrap" or "one paragraph per line". ### Tables @@ -30,7 +30,7 @@ Table: Caption for this example table. {#tbl:example-id} Support for table numbering and citation is provided by [`pandoc-tablenos`](https://github.com/tomduck/pandoc-tablenos). Above, `{#tbl:example-id}` sets the table ID, which creates an HTML anchor and allows citing the table like `@tbl:example-id`. -For easy creation of markdown tables, check out the [Tables Generator](http://www.tablesgenerator.com/markdown_tables) webapp. +For easy creation of markdown tables, check out the [Tables Generator](https://www.tablesgenerator.com/markdown_tables) webapp. ### Figures @@ -45,11 +45,11 @@ This figure can be cited in the text using `@fig:example-id`. In context, a figure citation may look like: `Figure {@fig:example-id}B shows …`. For images created by the manuscript authors that are hosted elsewhere on GitHub, we recommend using a [versioned](https://help.github.com/articles/getting-permanent-links-to-files/) GitHub URL to embed figures, thereby preserving exact image provenance. -When embedding SVG images hosted on GitHub, passing the URL through [RawGit](https://rawgit.com/) is necessary. -An example of a URL that has been passed through RawGit is: +When embedding SVG images hosted on GitHub, it's necessary to append `?sanitize=true` to the `raw.githubusercontent.com` URL. +For example: ``` -https://cdn.rawgit.com/greenelab/scihub/572d6947cb958e797d1a07fdb273157ad9154273/figure/citescore.svg +https://raw.githubusercontent.com/greenelab/scihub/572d6947cb958e797d1a07fdb273157ad9154273/figure/citescore.svg?sanitize=true ``` Figures placed in the [`content/images`](content/images) directory can be embedded using their relative path. @@ -59,19 +59,19 @@ For example, we embed an [ORCID](https://orcid.org/) icon inline using: ![ORCID icon](images/orcid.svg){height="13px"} ``` -The bracketed text following the image declaration is interpreted by Pandoc's [`link_attributes`](http://pandoc.org/MANUAL.html#extension-link_attributes) extension. +The bracketed text following the image declaration is interpreted by Pandoc's [`link_attributes`](https://pandoc.org/MANUAL.html#extension-link_attributes) extension. For example, the following will override the figure number to be "S1" and set the image width to 5 inches: ```md {#fig:supplement tag="S1" width="5in"} ``` -We recommend always specifying the width of SVG images (even if just `width="100%"`), since otherwise SVGs may not render properly in the [WeasyPrint](http://weasyprint.org/) PDF export. +We recommend always specifying the width of SVG images (even if just `width="100%"`), since otherwise SVGs may not render properly in the [WeasyPrint](https://weasyprint.org/) PDF export. ### Citations -Manubot supports Pandoc [citations](http://pandoc.org/MANUAL.html#citations) via `pandoc-citeproc`. -However, Manubot performs automated citation processing and metadata retrieval on raw citations. +Manubot supports Pandoc [citations](https://pandoc.org/MANUAL.html#citations) via `pandoc-citeproc`. +However, Manubot performs automated citation processing and metadata retrieval on in-text citations. Therefore, citations must be of the following form: `@source:identifier`, where `source` is one of the options described below. When choosing which source to use for a citation, we recommend the following order: @@ -80,6 +80,8 @@ When choosing which source to use for a citation, we recommend the following ord 3. PubMed ID, cite like `@pmid:26158728`. 4. _arXiv_ ID, cite like `@arxiv:1508.06576v2`. 5. URL / webpage, cite like `@url:http://openreview.net/pdf?id=Sk-oDY9ge`. +6. For references that do not have any of the persistent identifiers above, use a raw citation like `@raw:old-manuscript`. +Metadata for raw citations must be provided manually. Cite multiple items at once like: @@ -109,8 +111,8 @@ The Manubot workflow requires the bibliographic details for references (the set The Manubot attempts to automatically retrieve metadata and generate valid citeproc JSON for references, which is exported to `output/references.json`. However, in some cases the Manubot fails to retrieve metadata or generates incorrect or incomplete citeproc metadata. Errors are most common for `url` references. -For these references, you can manually specify the correct citeproc in [`content/manual-references.json`](content/manual-references.json), which will override the automatically generated reference data. -To do so, create a new citeproc record that contains the field `"standard_citation"` with the appropriate reference identifier as its value. +For these references, you can manually specify the correct CSL Data in [`content/manual-references.json`](content/manual-references.json), which will override the automatically generated reference data. +To do so, create a new CSL JSON Item that contains the field `"standard_citation"` with the appropriate reference identifier as its value. The identifier can be obtained from the `standard_citation` column of `citations.tsv`, which is located in the `output` branch or in the `output` subdirectory of local builds. As an example, `manual-references.json` contains: @@ -118,11 +120,23 @@ As an example, `manual-references.json` contains: "standard_citation": "url:https://github.com/greenelab/manubot-rootstock" ``` +The metadata for `raw` citations must be provided in `manual-references.json` or an error will occur. +For example, to cite `@raw:private-message` in a manuscript, a corresponding CSL Item in `manual-references.json` is required, such as: + +```json +{ + "type": "personal_communication", + "standard_citation": "raw:private-message", + "title": "Personal communication with Doctor X" +} +``` + +All references provided in `manual-references.json` must provide values for the `type` and `standard_citation` fields. For guidance on what CSL JSON should be like for different document types, refer to [these examples](https://github.com/aurimasv/zotero-import-export-formats/blob/a51c342e66bebd97b73a7230047b801c8f7bb690/CSL%20JSON.json). ## Manuscript metadata -[`content/metadata.yaml`](content/metadata.yaml) contains manuscript metadata that gets passed through to Pandoc, via a [`yaml_metadata_block`](http://pandoc.org/MANUAL.html#extension-yaml_metadata_block). +[`content/metadata.yaml`](content/metadata.yaml) contains manuscript metadata that gets passed through to Pandoc, via a [`yaml_metadata_block`](https://pandoc.org/MANUAL.html#extension-yaml_metadata_block). `metadata.yaml` should contain the manuscript `title`, `authors` list, `keywords`, and `lang` ([language tag](https://www.w3.org/International/articles/language-tags/ "W3C: Language tags in HTML and XML")). Additional metadata, such as `date`, will automatically be created by the Manubot. Manubot uses the [timezone](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) specified in [`build.sh`](build/build.sh) for setting the manuscript's date. diff --git a/build/environment.yml b/build/environment.yml index 73171db..77d7a50 100644 --- a/build/environment.yml +++ b/build/environment.yml @@ -15,7 +15,7 @@ dependencies: - conda-forge::watchdog=0.8.3 - pip: - errorhandler==2.0.1 - - git+https://github.com/greenelab/manubot@4e6a0f6d28220264c0d7892c732cb68c3e97c07a + - git+https://github.com/greenelab/manubot@a008126b39e3bd4b80ebaa5af9f9fa2f30b3a670 - jsonref==0.2 - opentimestamps-client==0.6.0 - opentimestamps==0.3.0 diff --git a/content/02.delete-me.md b/content/02.delete-me.md index ba79516..6484a4e 100644 --- a/content/02.delete-me.md +++ b/content/02.delete-me.md @@ -6,10 +6,10 @@ The Manubot is a system for automating scholarly publishing. Content is written in [Pandoc Markdown](http://pandoc.org/MANUAL.html#pandocs-markdown) source files. See [`USAGE.md`](https://github.com/greenelab/manubot-rootstock/blob/master/USAGE.md) for more information on how to use the Manubot. -The Manubot project began with the [Deep Review](https://github.com/greenelab/deep-review), where it was used to compose a highly-collaborative review article [@doi:10.1101/142760]. +The Manubot project began with the [Deep Review](https://github.com/greenelab/deep-review), where it was used to compose a highly-collaborative review article [@doi:10.1098/rsif.2017.0387]. Other manuscripts that were created with Manubot include: -+ The Sci-Hub Coverage Study ([GitHub](https://github.com/greenelab/scihub-manuscript), [HTML manuscript](https://greenelab.github.io/scihub-manuscript/)) ++ The Sci-Hub Coverage Study ([GitHub](https://github.com/greenelab/scihub-manuscript), [HTML manuscript](https://greenelab.github.io/scihub-manuscript/)) [@doi:10.7554/eLife.32822] + Michael Zietz's Report for the Vagelos Scholars Program ([GitHub](https://github.com/zietzm/Vagelos2017), [HTML manuscript](https://zietzm.github.io/Vagelos2017/)) [@doi:10.6084/m9.figshare.5346577] If you notice a problem with Manubot, it's best to submit an upstream fix to the appropriate repository: From 135f45193f2bd1f6ce2aa95e06973d19073286d5 Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Fri, 2 Nov 2018 13:23:22 -0400 Subject: [PATCH 193/216] Ignore OS specific files Merges https://github.com/greenelab/manubot-rootstock/pull/142 --- .gitignore | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.gitignore b/.gitignore index 6befc7c..ec87fe3 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,17 @@ __pycache__/ # Misc temporary files *.bak +# Operating system specific files + +## Linux +*~ +.Trash-* + +## macOS +.DS_Store +._* +.Trashes + +## Windows +Thumbs.db +[Dd]esktop.ini From 44c5d35b86266606b22fdcfcb11d1bb064cda5c9 Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Fri, 9 Nov 2018 16:26:22 -0500 Subject: [PATCH 194/216] Support BUILD_PDF=false environment variable Merges https://github.com/greenelab/manubot-rootstock/pull/145 Closes https://github.com/greenelab/manubot-rootstock/issues/122 * Add BUILD_PDF environment variable BUILD_PDF is considered "true" unless specifically set to "false" webpage.py: tolerate missing output/manuscript.pdf Refs https://github.com/greenelab/manubot-rootstock/issues/144#issuecomment-437119645 ots stamp in deploy.sh: do not fail if BUILD_PDF=false * Delete existing images symlink in PDF build script When an images directory exists, do not create a symbolic link with source images/images. More info in ln documentation: - https://ss64.com/osx/ln.html - https://linux.die.net/man/1/ln Ignore /images temporary symlink --- .gitignore | 4 ++++ build/README.md | 5 ++++- build/build.sh | 50 +++++++++++++++++++++++++----------------------- build/webpage.py | 5 ++++- ci/deploy.sh | 7 ++++--- 5 files changed, 42 insertions(+), 29 deletions(-) diff --git a/.gitignore b/.gitignore index ec87fe3..92941be 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,10 @@ output/* webpage/v +# When PDF building fails, a temporary symlink named images in the root +# directory is not removed. +/images + # Manubot cache directory ci/cache diff --git a/build/README.md b/build/README.md index 98f1981..86f9d82 100644 --- a/build/README.md +++ b/build/README.md @@ -2,10 +2,13 @@ [`build.sh`](build.sh) builds the repository. `sh build/build.sh` should be executed from the root directory of the repository. +By default, `build.sh` creates HTML and PDF outputs. +However, setting the `BUILD_PDF` environment variable to `false` will suppress PDF output. +For example, run local builds using the command `BUILD_PDF=false sh build/build.sh`. To build a DOCX file of the manuscript, set the `BUILD_DOCX` environment variable to `true`. For example, use the command `BUILD_DOCX=true sh build/build.sh`. -Set a [Travis environment variable](https://docs.travis-ci.com/user/environment-variables/#Defining-Variables-in-Repository-Settings) to export DOCX for all Travis builds. +To export DOCX for all Travis builds, set a [Travis environment variable](https://docs.travis-ci.com/user/environment-variables/#Defining-Variables-in-Repository-Settings). Currently, equation numbers via `pandoc-eqnos` are not supported for DOCX output. There is varying support for embedding images in DOCX output. Please reference [Pull Request #40](https://github.com/greenelab/manubot-rootstock/pull/40) for possible solutions and continued discussion. diff --git a/build/build.sh b/build/build.sh index 60e4adb..8ff318d 100644 --- a/build/build.sh +++ b/build/build.sh @@ -42,31 +42,33 @@ pandoc --verbose \ --output=output/manuscript.html \ $INPUT_PATH -# Create PDF output -echo "Exporting PDF manuscript" -ln -s content/images images -pandoc \ - --from=markdown \ - --to=html5 \ - --pdf-engine=weasyprint \ - --pdf-engine-opt=--presentational-hints \ - --filter=pandoc-fignos \ - --filter=pandoc-eqnos \ - --filter=pandoc-tablenos \ - --bibliography=$BIBLIOGRAPHY_PATH \ - --csl=$CSL_PATH \ - --metadata link-citations=true \ - --webtex=https://latex.codecogs.com/svg.latex? \ - --css=webpage/github-pandoc.css \ - --output=output/manuscript.pdf \ - $INPUT_PATH -rm -r images +# Create PDF output (unless BUILD_PDF environment variable equals "false") +if [ "$BUILD_PDF" != "false" ]; then + echo "Exporting PDF manuscript" + if [ -L images ]; then rm images; fi # if images is a symlink, remove it + ln -s content/images + pandoc \ + --from=markdown \ + --to=html5 \ + --pdf-engine=weasyprint \ + --pdf-engine-opt=--presentational-hints \ + --filter=pandoc-fignos \ + --filter=pandoc-eqnos \ + --filter=pandoc-tablenos \ + --bibliography=$BIBLIOGRAPHY_PATH \ + --csl=$CSL_PATH \ + --metadata link-citations=true \ + --webtex=https://latex.codecogs.com/svg.latex? \ + --css=webpage/github-pandoc.css \ + --output=output/manuscript.pdf \ + $INPUT_PATH + rm images +fi -# Create DOCX output when user specifies to do so -if [ "$BUILD_DOCX" = "true" ]; -then - echo "Exporting Word Docx manuscript" - pandoc --verbose \ +# Create DOCX output (if BUILD_DOCX environment variable equals "true") +if [ "$BUILD_DOCX" = "true" ]; then + echo "Exporting Word Docx manuscript" + pandoc --verbose \ --from=markdown \ --to=docx \ --filter=pandoc-fignos \ diff --git a/build/webpage.py b/build/webpage.py index bf0b021..31fc287 100644 --- a/build/webpage.py +++ b/build/webpage.py @@ -117,8 +117,11 @@ def create_version(args): 'manuscript.pdf': 'manuscript.pdf', } for src, dst in renamer.items(): + src_path = args.output_directory.joinpath(src) + if not src_path.exists(): + continue shutil.copy2( - src=args.output_directory.joinpath(src), + src=src_path, dst=args.version_directory.joinpath(dst), ) diff --git a/ci/deploy.sh b/ci/deploy.sh index 96e40f0..e349cc8 100644 --- a/ci/deploy.sh +++ b/ci/deploy.sh @@ -35,9 +35,10 @@ python build/webpage.py \ --version=$TRAVIS_COMMIT # Generate OpenTimestamps -ots stamp \ - webpage/v/$TRAVIS_COMMIT/index.html \ - webpage/v/$TRAVIS_COMMIT/manuscript.pdf +ots stamp webpage/v/$TRAVIS_COMMIT/index.html +if [ "$BUILD_PDF" != "false" ]; then + ots stamp webpage/v/$TRAVIS_COMMIT/manuscript.pdf +fi # Commit message MESSAGE="\ From 66214ac0e7caf67908fa427cbb4fd894c55f23ef Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Mon, 12 Nov 2018 16:59:31 -0500 Subject: [PATCH 195/216] Add pango to conda environment Closes https://github.com/greenelab/manubot-rootstock/issues/144 Merges https://github.com/greenelab/manubot-rootstock/pull/146 pango is required by WeasyPrint during the PDF build. While usually built-in to the OS on Linux, it is not pre-installed on macOS. macOS were installing pango using homebrew, which led to a segmentation fault error. Pango is available on conda-forge for linux and mac: https://anaconda.org/conda-forge/pango https://github.com/conda-forge/pango-feedstock Adding pango to environment.yml prevents conda environment installation on Windows, but has the benefits of standardizing the pango version on linux and mac platforms and preventing the mac homebrew issue. --- build/README.md | 5 ++++- build/environment.yml | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/build/README.md b/build/README.md index 86f9d82..e79063b 100644 --- a/build/README.md +++ b/build/README.md @@ -15,6 +15,8 @@ Please reference [Pull Request #40](https://github.com/greenelab/manubot-rootsto ## Environment +Note: currently, **Windows is not supported**. + Install or update the [conda](https://conda.io) environment specified in [`environment.yml`](environment.yml) by running: ```sh @@ -26,4 +28,5 @@ conda env create --file environment.yml ``` Activate with `conda activate manubot` (assumes `conda` version of [at least](https://github.com/conda/conda/blob/9d759d8edeb86569c25f6eb82053f09581013a2a/CHANGELOG.md#440-2017-12-20) 4.4). -The environment should successfully install on both Linux and macOS (and possibly Windows). +The environment should successfully install on both Linux and macOS. +However, it will fail on Windows due to the [`pango`](https://anaconda.org/conda-forge/pango) dependency. diff --git a/build/environment.yml b/build/environment.yml index 77d7a50..c1d4bc7 100644 --- a/build/environment.yml +++ b/build/environment.yml @@ -9,6 +9,7 @@ dependencies: - conda-forge::jsonschema=2.6.0 - conda-forge::pandas=0.23.4 - conda-forge::pandoc=2.3.1 + - conda-forge::pango=1.40.14 - conda-forge::python=3.6.5 - conda-forge::pyyaml=3.13 - conda-forge::requests=2.19.1 From 4e68632bf2d7126fec1595642021a1712bed11e0 Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Thu, 15 Nov 2018 14:43:06 -0500 Subject: [PATCH 196/216] Update manubot to v0.2.1 with PMCID fix Merges https://github.com/greenelab/manubot-rootstock/pull/147 Update manubot version on 2018-11-15 to https://github.com/greenelab/manubot/commit/afa55f83a4a27645b07bdf0a25092a45b814df35 Includes migration of PMCID citations to Literature Citation Exporter. Update conda-forge dependencies --- build/environment.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/environment.yml b/build/environment.yml index c1d4bc7..4983806 100644 --- a/build/environment.yml +++ b/build/environment.yml @@ -12,11 +12,11 @@ dependencies: - conda-forge::pango=1.40.14 - conda-forge::python=3.6.5 - conda-forge::pyyaml=3.13 - - conda-forge::requests=2.19.1 + - conda-forge::requests=2.20.1 - conda-forge::watchdog=0.8.3 - pip: - errorhandler==2.0.1 - - git+https://github.com/greenelab/manubot@a008126b39e3bd4b80ebaa5af9f9fa2f30b3a670 + - git+https://github.com/greenelab/manubot@afa55f83a4a27645b07bdf0a25092a45b814df35 - jsonref==0.2 - opentimestamps-client==0.6.0 - opentimestamps==0.3.0 From a4428b50200987ee8a6ddfb63dacc083f6248f7f Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Fri, 16 Nov 2018 09:56:18 -0500 Subject: [PATCH 197/216] SETUP.py: add checkout command to rootstock upgrade Merges https://github.com/greenelab/manubot-rootstock/pull/148 Co-Authored-By: Daniel Himmelstein Co-Authored-By: Anthony Gitter --- SETUP.md | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/SETUP.md b/SETUP.md index 22a4f06..a32ae13 100644 --- a/SETUP.md +++ b/SETUP.md @@ -174,7 +174,15 @@ This section will describe how to incorporate changes to manubot-rootstock that You will want to do this if there are new enhancements or bugfixes that you want to incorporate. This process can be difficult, especially if conflicts have arisen, and is recommended only for advanced git users. -First, pull the new commits from manubot-rootstock, but do not automerge: +It is recommended to do rootstock upgrades via a pull request to help you view the proposed changes and to ensure the build uses the updated environment. +First, checkout a new branch to use as the pull request head branch: + +```sh +# This command names the branch using the current date, i.e. rootstock-2018-11-16 +git checkout -b rootstock-`date '+%Y-%m-%d'` +``` + +Second, pull the new commits from manubot-rootstock, but do not automerge: ```sh git pull --no-ff --no-commit rootstock master @@ -185,6 +193,9 @@ However, if there are conflicts, follow the suggested commands to resolve them. You can add the changes incrementally using `git add --patch`. This is helpful to see each upstream change. -You may notice changes that effect how items in `content` are processed. +You may notice changes that affect how items in `content` are processed. If so, you should edit and stage `content` files as needed. When there are no longer any unstaged changes, then do `git commit`. + +If updating `master` via a pull request, proceed to push the commit to GitHub and open a pull request. +Once the pull request is ready to merge, use GitHub's "Create a merge commit" option rather than "Squash and merge" or "Rebase and merge" to preserve the rootstock commit hashes. From 1614fa305d0850ac3756edabc1cb4d3dfae5f346 Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Tue, 20 Nov 2018 11:06:40 -0500 Subject: [PATCH 198/216] Enable ISBN citation support Merges https://github.com/greenelab/manubot-rootstock/pull/149 * Enable ISBN citaiton support * Update URL citation Previous URL citation contained "?", therefore requiring a citation tag. --- USAGE.md | 7 ++++--- build/environment.yml | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/USAGE.md b/USAGE.md index 5bd3507..81fd0b0 100644 --- a/USAGE.md +++ b/USAGE.md @@ -79,14 +79,15 @@ When choosing which source to use for a citation, we recommend the following ord 2. PubMed Central ID, cite like `@pmcid:PMC4497619`. 3. PubMed ID, cite like `@pmid:26158728`. 4. _arXiv_ ID, cite like `@arxiv:1508.06576v2`. -5. URL / webpage, cite like `@url:http://openreview.net/pdf?id=Sk-oDY9ge`. -6. For references that do not have any of the persistent identifiers above, use a raw citation like `@raw:old-manuscript`. +5. ISBN (International Standard Book Number), cite like `@isbn:9781339919881`. +6. URL / webpage, cite like `@url:https://nyti.ms/1QUgAt1`. +7. For references that do not have any of the persistent identifiers above, use a raw citation like `@raw:old-manuscript`. Metadata for raw citations must be provided manually. Cite multiple items at once like: ```md -Here is a sentence with several citations [@doi:10.15363/thinklab.4; @pmid:26158728; @arxiv:1508.06576]. +Here is a sentence with several citations [@doi:10.15363/thinklab.4; @pmid:26158728; @arxiv:1508.06576; @isbn:9780394603988]. ``` Note that multiple citations must be semicolon separated. diff --git a/build/environment.yml b/build/environment.yml index 4983806..9b58958 100644 --- a/build/environment.yml +++ b/build/environment.yml @@ -16,7 +16,7 @@ dependencies: - conda-forge::watchdog=0.8.3 - pip: - errorhandler==2.0.1 - - git+https://github.com/greenelab/manubot@afa55f83a4a27645b07bdf0a25092a45b814df35 + - git+https://github.com/greenelab/manubot@3318c8f220f557d3f6603df28b2b4db3ead7ecda - jsonref==0.2 - opentimestamps-client==0.6.0 - opentimestamps==0.3.0 From aa664b4fa8974ef2f57f73593d781173b24243ea Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Tue, 11 Dec 2018 11:06:20 -0500 Subject: [PATCH 199/216] webpage utility: upgrade preexisting timestamps Merges https://github.com/greenelab/manubot-rootstock/pull/150 Attempt to upgrade preexisting timestamps during continuous deployment, such that manually upgrading incomplete timestamps should no longer be necessary. Add two mutually exclusive arguments to webpage.py: --ots-cache and --no-ots-cache for configuring the OTS client caching. Disable timestamp caching for deploy.sh (i.e. on Travis CI) --- build/webpage.py | 63 ++++++++++++++++++++++++++++++++++++++++++++--- ci/deploy.sh | 1 + webpage/README.md | 8 ++++-- 3 files changed, 67 insertions(+), 5 deletions(-) diff --git a/build/webpage.py b/build/webpage.py index 31fc287..f6191e9 100644 --- a/build/webpage.py +++ b/build/webpage.py @@ -18,7 +18,21 @@ def parse_arguments(): parser.add_argument( '--version', default=os.environ.get('TRAVIS_COMMIT', 'local'), - help="Used to create webpage/v/{version} directory. Generally a commit hash, tag, or 'local'", + help="Used to create webpage/v/{version} directory. " + "Generally a commit hash, tag, or 'local'. " + "(default: '%(default)s')" + ) + cache_group = parser.add_mutually_exclusive_group() + cache_group.add_argument( + '--no-ots-cache', + action='store_true', + help="disable the timestamp cache." + ) + cache_group.add_argument( + '--ots-cache', + default=pathlib.Path('ci/cache/ots'), + type=pathlib.Path, + help="location for the timestamp cache (default: %(default)s)." ) args = parser.parse_args() return args @@ -138,7 +152,7 @@ def create_version(args): args.freeze_directory.joinpath('index.html').write_text(redirect_html) -def get_versions(): +def get_versions(args): """ Extract versions from the webpage/v directory, which should each contain a manuscript. @@ -149,10 +163,53 @@ def get_versions(): return versions +def ots_upgrade(args): + """ + Upgrade OpenTimestamps .ots files in versioned commit directory trees. + + Upgrades each .ots file with a separate ots upgrade subprocess call due to + https://github.com/opentimestamps/opentimestamps-client/issues/71 + """ + ots_paths = list() + for version in get_versions(args): + ots_paths.extend(args.versions_directory.joinpath(version).glob('**/*.ots')) + ots_paths.sort() + for ots_path in ots_paths: + process_args = ['ots'] + if args.no_ots_cache: + process_args.append('--no-cache') + else: + process_args.extend(['--cache', str(args.ots_cache)]) + process_args.extend([ + 'upgrade', + str(ots_path), + ]) + process = subprocess.run( + process_args, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + if process.returncode != 0: + print(f"OpenTimestamp upgrade command returned nonzero code ({process.returncode}).") + if not process.stderr.strip() == 'Success! Timestamp complete': + print( + f">>> {' '.join(map(str, process.args))}\n" + f"{process.stderr}" + ) + backup_path = ots_path.with_suffix('.ots.bak') + if backup_path.exists(): + if process.returncode == 0: + backup_path.unlink() + else: + # Restore original timestamp if failure + backup_path.rename(ots_path) + + if __name__ == '__main__': args = parse_arguments() configure_directories(args) print(args) create_version(args) - versions = get_versions() + versions = get_versions(args) print(versions) + ots_upgrade(args) diff --git a/ci/deploy.sh b/ci/deploy.sh index e349cc8..e91a254 100644 --- a/ci/deploy.sh +++ b/ci/deploy.sh @@ -31,6 +31,7 @@ git fetch origin gh-pages:gh-pages output:output # Configure versioned webpage python build/webpage.py \ + --no-ots-cache \ --checkout=gh-pages \ --version=$TRAVIS_COMMIT diff --git a/webpage/README.md b/webpage/README.md index b347ecc..6153fe5 100644 --- a/webpage/README.md +++ b/webpage/README.md @@ -20,15 +20,19 @@ In general, a version is identified by the commit hash of the source content tha The `*.ots` files in version directories are OpenTimestamps which can be used to verify manuscript existence at or before a given time. [OpenTimestamps](https://opentimestamps.org/) uses the Bitcoin blockchain to attest to file hash existence. The `deploy.sh` script run during continuous deployment creates the `.ots` files. -However, these files are initially dependent on calendar services and must be upgraded at a later time by running the following in the `gh-pages` branch: +There is a delay before timestamps get confirmed by a Bitcoin block. +Therefore, `.ots` files are initially incomplete and should be upgraded at a later time, so that they no longer rely on the availability of a calendar server to verify. +`webpage.py`, which is run during continuous deployment, identifies files matched by `webpage/v/**/*.ots` and attempts to upgrade them. +You can also manually upgrade timestamps, by running the following in the `gh-pages` branch: ```sh -# Requires a local bitcoin node with JSON-RPC configured ots upgrade v/*/*.ots rm v/*/*.ots.bak git add v/*/*.ots ``` +Verifying timestamps with the `ots verify` command requires running a local bitcoin node with JSON-RPC configured, at this time. + ## Source The manuscripts in this directory were built from From 9e6a00b51ae0925118104de952cadac9ce91c75f Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Tue, 18 Dec 2018 12:13:34 -0500 Subject: [PATCH 200/216] Upgrades on 2018-12-18 including wikidata citation support Merges https://github.com/greenelab/manubot-rootstock/pull/151 * Upgrade manubot python package with Wikidata citations * Remove DataCite DOI from 02.delete-me.md * Add additional manuscripts to USAGE examples --- USAGE.md | 12 ++++++++---- build/environment.yml | 2 +- content/02.delete-me.md | 3 +-- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/USAGE.md b/USAGE.md index 81fd0b0..af038b6 100644 --- a/USAGE.md +++ b/USAGE.md @@ -81,8 +81,10 @@ When choosing which source to use for a citation, we recommend the following ord 4. _arXiv_ ID, cite like `@arxiv:1508.06576v2`. 5. ISBN (International Standard Book Number), cite like `@isbn:9781339919881`. 6. URL / webpage, cite like `@url:https://nyti.ms/1QUgAt1`. -7. For references that do not have any of the persistent identifiers above, use a raw citation like `@raw:old-manuscript`. -Metadata for raw citations must be provided manually. +7. Wikidata Items, cite like `@wikidata:Q50051684`. + Note that anyone can edit or add records on [Wikidata](https://www.wikidata.org), so users are encouraged to contribute metadata for hard-to-cite works to Wikidata as an alternative to using a `raw` citation. +8. For references that do not have any of the persistent identifiers above, use a raw citation like `@raw:old-manuscript`. + Metadata for raw citations must be provided manually. Cite multiple items at once like: @@ -99,9 +101,9 @@ For example, the following citations all refer to the same study, but will be tr The system also supports citation tags, which are recommended for the following applications: 1. A citation's identifier contains forbidden characters, such as `;` or `=`, or ends with a non-alphanumeric character other than `/`. - In these instances, you must use a tag. + In these instances, you must use a tag. 2. A single reference is cited many times. - Therefore, it might make sense to define a tag, so if the citation updates (e.g. a newer version becomes available), only a single change is required. + Therefore, it might make sense to define a tag, so if the citation updates (e.g. a newer version becomes available), only a single change is required. Tags should be defined in [`content/citation-tags.tsv`](content/citation-tags.tsv). If `citation-tags.tsv` defines the tag `study-x`, then this study can be cited like `@tag:study-x`. @@ -171,9 +173,11 @@ For additional examples, check out existing manuscripts that use the Manubot (so + Satoshi Nakamoto's Bitcoin Whitepaper ([source](https://github.com/dhimmel/bitcoin-whitepaper/), [manuscript](https://dhimmel.github.io/bitcoin-whitepaper/), [commentary](https://steemit.com/manubot/@dhimmel/how-i-used-the-manubot-to-reproduce-the-bitcoin-whitepaper)) + The Sci-Hub Coverage Study ([source](https://github.com/greenelab/scihub-manuscript), [manuscript](https://greenelab.github.io/scihub-manuscript/)) ++ The GimmeMotifs manscript on transcription factor motif analysis ([source](https://github.com/simonvh/gimmemotifs-manuscript), [manuscript](https://simonvh.github.io/gimmemotifs-manuscript/manuscript.pdf)) + A Report for the Vagelos Scholars Program by Michael Zietz ([source](https://github.com/zietzm/Vagelos2017), [manuscript](https://zietzm.github.io/Vagelos2017/)) + The Deep Review ([source](https://github.com/greenelab/deep-review), [manuscript](https://greenelab.github.io/deep-review/)) + The Meta Review ([source](https://github.com/greenelab/meta-review), [manuscript](https://greenelab.github.io/meta-review/)) ++ A review of Network Methods for Multiomic Data Integration ([source](https://github.com/zietzm/integration-review), [manuscript](https://zietzm.github.io/integration-review/)) + The Project Rephetio Manuscript ([source](https://github.com/dhimmel/rephetio-manuscript/), [manuscript](https://dhimmel.github.io/rephetio-manuscript/)) + A Literature Review for Project Planning by David Slochower ([source](https://github.com/slochower/synthetic-motor-literature), [manuscript](https://slochower.github.io/synthetic-motor-literature/)) + The TFSEE Manuscript by Venkat Malladi et al. ([source](https://github.com/vsmalladi/tfsee-manuscript), [manuscript](https://vsmalladi.github.io/tfsee-manuscript/)) diff --git a/build/environment.yml b/build/environment.yml index 9b58958..f815add 100644 --- a/build/environment.yml +++ b/build/environment.yml @@ -16,7 +16,7 @@ dependencies: - conda-forge::watchdog=0.8.3 - pip: - errorhandler==2.0.1 - - git+https://github.com/greenelab/manubot@3318c8f220f557d3f6603df28b2b4db3ead7ecda + - git+https://github.com/greenelab/manubot@dae57036de250ca1ece852a4f5b3d7f946c25d50 - jsonref==0.2 - opentimestamps-client==0.6.0 - opentimestamps==0.3.0 diff --git a/content/02.delete-me.md b/content/02.delete-me.md index 6484a4e..9f0e403 100644 --- a/content/02.delete-me.md +++ b/content/02.delete-me.md @@ -7,10 +7,9 @@ Content is written in [Pandoc Markdown](http://pandoc.org/MANUAL.html#pandocs-ma See [`USAGE.md`](https://github.com/greenelab/manubot-rootstock/blob/master/USAGE.md) for more information on how to use the Manubot. The Manubot project began with the [Deep Review](https://github.com/greenelab/deep-review), where it was used to compose a highly-collaborative review article [@doi:10.1098/rsif.2017.0387]. -Other manuscripts that were created with Manubot include: +Another example manuscript that was created with Manubot is: + The Sci-Hub Coverage Study ([GitHub](https://github.com/greenelab/scihub-manuscript), [HTML manuscript](https://greenelab.github.io/scihub-manuscript/)) [@doi:10.7554/eLife.32822] -+ Michael Zietz's Report for the Vagelos Scholars Program ([GitHub](https://github.com/zietzm/Vagelos2017), [HTML manuscript](https://zietzm.github.io/Vagelos2017/)) [@doi:10.6084/m9.figshare.5346577] If you notice a problem with Manubot, it's best to submit an upstream fix to the appropriate repository: [`greenelab/manubot-rootstock`](https://github.com/greenelab/manubot-rootstock) for the git repository stub or [`greenelab/manubot`](https://github.com/greenelab/manubot) for the Python package. From 4103ad39859e2d0e02e72ec204fbd7b80d12aeb4 Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Thu, 3 Jan 2019 13:36:22 -1000 Subject: [PATCH 201/216] Update environment & examples on 2019-01-03 Merges https://github.com/greenelab/manubot-rootstock/pull/153 Updates pandoc to v2.5. --- USAGE.md | 2 ++ build/environment.yml | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/USAGE.md b/USAGE.md index af038b6..2379e7b 100644 --- a/USAGE.md +++ b/USAGE.md @@ -176,6 +176,7 @@ For additional examples, check out existing manuscripts that use the Manubot (so + The GimmeMotifs manscript on transcription factor motif analysis ([source](https://github.com/simonvh/gimmemotifs-manuscript), [manuscript](https://simonvh.github.io/gimmemotifs-manuscript/manuscript.pdf)) + A Report for the Vagelos Scholars Program by Michael Zietz ([source](https://github.com/zietzm/Vagelos2017), [manuscript](https://zietzm.github.io/Vagelos2017/)) + The Deep Review ([source](https://github.com/greenelab/deep-review), [manuscript](https://greenelab.github.io/deep-review/)) ++ Ten Quick Tips for Deep Learning ([source](https://github.com/Benjamin-Lee/deep-rules), [manuscript](https://benjamin-lee.github.io/deep-rules/)) + The Meta Review ([source](https://github.com/greenelab/meta-review), [manuscript](https://greenelab.github.io/meta-review/)) + A review of Network Methods for Multiomic Data Integration ([source](https://github.com/zietzm/integration-review), [manuscript](https://zietzm.github.io/integration-review/)) + The Project Rephetio Manuscript ([source](https://github.com/dhimmel/rephetio-manuscript/), [manuscript](https://dhimmel.github.io/rephetio-manuscript/)) @@ -183,6 +184,7 @@ For additional examples, check out existing manuscripts that use the Manubot (so + The TFSEE Manuscript by Venkat Malladi et al. ([source](https://github.com/vsmalladi/tfsee-manuscript), [manuscript](https://vsmalladi.github.io/tfsee-manuscript/)) + Creating a Global Emissions Timeseries dataset by Robert Gieseke et al. ([source](https://github.com/openclimatedata/global-emissions), [manuscript](https://openclimatedata.github.io/global-emissions/)) + The yt 3.0 methods paper ([source](https://github.com/yt-project/yt-3.0-paper), [manuscript](https://yt-project.github.io/yt-3.0-paper/)) ++ The TPOT-DS Manuscript (includes Hypothesis annotations, [source](https://github.com/trang1618/tpot-ds-ms), [manuscript](https://trang1618.github.io/tpot-ds-ms/)) + The Manubot 2018 Development Proposal ([source](https://github.com/greenelab/manufund-2018), [manuscript](https://greenelab.github.io/manufund-2018/)) If you are using the Manubot, feel free to submit a pull request to add your manuscript to the list above. diff --git a/build/environment.yml b/build/environment.yml index f815add..60e6e73 100644 --- a/build/environment.yml +++ b/build/environment.yml @@ -8,15 +8,15 @@ dependencies: - conda-forge::jinja2=2.10 - conda-forge::jsonschema=2.6.0 - conda-forge::pandas=0.23.4 - - conda-forge::pandoc=2.3.1 + - conda-forge::pandoc=2.5 - conda-forge::pango=1.40.14 - conda-forge::python=3.6.5 - conda-forge::pyyaml=3.13 - - conda-forge::requests=2.20.1 + - conda-forge::requests=2.21.0 - conda-forge::watchdog=0.8.3 - pip: - errorhandler==2.0.1 - - git+https://github.com/greenelab/manubot@dae57036de250ca1ece852a4f5b3d7f946c25d50 + - git+https://github.com/greenelab/manubot@552dd99aaa148365a29a87176fedd626d4642d3c - jsonref==0.2 - opentimestamps-client==0.6.0 - opentimestamps==0.3.0 From b1b274de8027dcd4db102cddad2c00f6d614b8f6 Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Mon, 21 Jan 2019 16:36:39 -0500 Subject: [PATCH 202/216] front-matter: omit null or empty author fields Merges https://github.com/greenelab/manubot-rootstock/pull/161 --- content/00.front-matter.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/content/00.front-matter.md b/content/00.front-matter.md index c438b3d..f8d4e32 100644 --- a/content/00.front-matter.md +++ b/content/00.front-matter.md @@ -26,23 +26,23 @@ on {{date}}. {## Template for listing authors ##} {% for author in authors %} + **{{author.name}}**
- {%- if author.orcid is defined %} + {%- if author.orcid is defined and author.orcid is not none %} ![ORCID icon](images/orcid.svg){height="13px" width="13px"} [{{author.orcid}}](https://orcid.org/{{author.orcid}}) {%- endif %} - {%- if author.github is defined %} + {%- if author.github is defined and author.github is not none %} · ![GitHub icon](images/github.svg){height="13px" width="13px"} [{{author.github}}](https://github.com/{{author.github}}) {%- endif %} - {%- if author.twitter is defined %} + {%- if author.twitter is defined and author.twitter is not none %} · ![Twitter icon](images/twitter.svg){height="13px" width="13px"} [{{author.twitter}}](https://twitter.com/{{author.twitter}}) {%- endif %}
- {%- if author.affiliations is defined %} + {%- if author.affiliations is defined and author.affiliations|length %} {{author.affiliations | join('; ')}} {%- endif %} - {%- if author.funders is defined %} + {%- if author.funders is defined and author.funders|length %} · Funded by {{author.funders}} {%- endif %} From eb4cb3efef46af8bfff1e391853006777cef373c Mon Sep 17 00:00:00 2001 From: Vincent Rubinetti Date: Tue, 22 Jan 2019 16:34:14 -0500 Subject: [PATCH 203/216] CSS improvements to github-pandoc.css Merges https://github.com/greenelab/manubot-rootstock/pull/159 When printing, hide hypothesis toolbar and fix page margins. Fix pandoc syntax highlighting quirk: long width code boxes overflow outside of container. Fix superscript size issue documented at https://github.com/greenelab/meta-review/pull/88#issuecomment-408546057 --- webpage/github-pandoc.css | 49 ++++++++++++++++++++++++++++++++------- 1 file changed, 40 insertions(+), 9 deletions(-) diff --git a/webpage/github-pandoc.css b/webpage/github-pandoc.css index e0e5e81..86154b5 100644 --- a/webpage/github-pandoc.css +++ b/webpage/github-pandoc.css @@ -58,7 +58,7 @@ template { */ body { - position: relative; + position: relative; } /* ========================================================================== @@ -85,13 +85,26 @@ body { margin: 0; } -/***Page size and margins when printing***/ +/** + * Page size and margins when printing. + */ @page { - size: Letter; - margin-top: 19mm; - margin-bottom: 16mm; - margin-left: 0mm; - margin-right: 0mm; + size: letter; + margin-top: 0.75in; + margin-bottom: 0.75in; + margin-left: 0.75in; + margin-right: 0.75in; +} +/* Remove margin, padding, border, etc. when printing; go right up to @page margins. */ +@media print { + html, + body { + margin: 0 !important; + padding: 0 !important; + border: 0 !important; + box-shadow: none !important; + max-width: none !important; + } } /* @@ -262,7 +275,7 @@ sup { } sup { - top: -0.5em; + top: -0.1em; } sub { @@ -289,6 +302,25 @@ svg:not(:root) { overflow: hidden; } +/** + * Hide hypothesis annotation toolbar, tooltip, etc. on print. + */ +@media only print { + .annotator-frame { + display: none !important; + } + hypothesis-adder { + display: none !important; + } +} + +/** + * Override pandoc generated CSS quirks. + */ +.sourceCode { + overflow: auto !important; +} + /* ========================================================================== Figures ========================================================================== */ @@ -754,7 +786,6 @@ pre tt { sup, sub, a.footnote { - font-size: 1.4ex; height: 0; line-height: 1; vertical-align: super; From fa7949f633ee819fd4724fcd1be6433acf2eb824 Mon Sep 17 00:00:00 2001 From: Vincent Rubinetti Date: Mon, 28 Jan 2019 11:10:50 -0500 Subject: [PATCH 204/216] Refactor HTML assets to build/themes & build/plugins Merges https://github.com/greenelab/manubot-rootstock/pull/168 Refs https://github.com/greenelab/manubot-rootstock/issues/163 --- build/build.sh | 13 ++++++------- build/{assets => plugins}/analytics.html | 0 build/{assets => plugins}/anchors.html | 0 build/{assets => plugins}/hypothesis.html | 0 .../pandoc-reference.docx => themes/default.docx} | Bin .../github-pandoc.css => build/themes/default.html | 2 ++ build/webpage.py | 6 ------ 7 files changed, 8 insertions(+), 13 deletions(-) rename build/{assets => plugins}/analytics.html (100%) rename build/{assets => plugins}/anchors.html (100%) rename build/{assets => plugins}/hypothesis.html (100%) rename build/{assets/pandoc-reference.docx => themes/default.docx} (100%) rename webpage/github-pandoc.css => build/themes/default.html (99%) diff --git a/build/build.sh b/build/build.sh index 8ff318d..69f6735 100644 --- a/build/build.sh +++ b/build/build.sh @@ -15,7 +15,6 @@ manubot process \ # pandoc settings CSL_PATH=build/assets/style.csl -DOCX_PATH=build/assets/pandoc-reference.docx BIBLIOGRAPHY_PATH=output/references.json INPUT_PATH=output/manuscript.md @@ -35,10 +34,10 @@ pandoc --verbose \ --csl=$CSL_PATH \ --metadata link-citations=true \ --mathjax \ - --css=github-pandoc.css \ - --include-in-header=build/assets/analytics.html \ - --include-after-body=build/assets/anchors.html \ - --include-after-body=build/assets/hypothesis.html \ + --include-after-body=build/themes/default.html \ + --include-after-body=build/plugins/analytics.html \ + --include-after-body=build/plugins/anchors.html \ + --include-after-body=build/plugins/hypothesis.html \ --output=output/manuscript.html \ $INPUT_PATH @@ -59,7 +58,7 @@ if [ "$BUILD_PDF" != "false" ]; then --csl=$CSL_PATH \ --metadata link-citations=true \ --webtex=https://latex.codecogs.com/svg.latex? \ - --css=webpage/github-pandoc.css \ + --include-after-body=build/themes/default.html \ --output=output/manuscript.pdf \ $INPUT_PATH rm images @@ -77,7 +76,7 @@ if [ "$BUILD_DOCX" = "true" ]; then --bibliography=$BIBLIOGRAPHY_PATH \ --csl=$CSL_PATH \ --metadata link-citations=true \ - --reference-doc=$DOCX_PATH \ + --reference-doc=build/themes/default.docx \ --resource-path=.:content \ --output=output/manuscript.docx \ $INPUT_PATH diff --git a/build/assets/analytics.html b/build/plugins/analytics.html similarity index 100% rename from build/assets/analytics.html rename to build/plugins/analytics.html diff --git a/build/assets/anchors.html b/build/plugins/anchors.html similarity index 100% rename from build/assets/anchors.html rename to build/plugins/anchors.html diff --git a/build/assets/hypothesis.html b/build/plugins/hypothesis.html similarity index 100% rename from build/assets/hypothesis.html rename to build/plugins/hypothesis.html diff --git a/build/assets/pandoc-reference.docx b/build/themes/default.docx similarity index 100% rename from build/assets/pandoc-reference.docx rename to build/themes/default.docx diff --git a/webpage/github-pandoc.css b/build/themes/default.html similarity index 99% rename from webpage/github-pandoc.css rename to build/themes/default.html index 86154b5..2bfcda0 100644 --- a/webpage/github-pandoc.css +++ b/build/themes/default.html @@ -1,3 +1,4 @@ + diff --git a/build/webpage.py b/build/webpage.py index f6191e9..101e6bd 100644 --- a/build/webpage.py +++ b/build/webpage.py @@ -139,12 +139,6 @@ def create_version(args): dst=args.version_directory.joinpath(dst), ) - # Copy webpage/github-pandoc.css to to webpage/v/version/github-pandoc.css - shutil.copy2( - src=args.webpage_directory.joinpath('github-pandoc.css'), - dst=args.version_directory.joinpath('github-pandoc.css'), - ) - # Create v/freeze to redirect to v/commit path = pathlib.Path('build/assets/redirect-template.html') redirect_html = path.read_text() From 96b580d853c3407a67f76a480364abe70a45099a Mon Sep 17 00:00:00 2001 From: Vincent Rubinetti Date: Mon, 28 Jan 2019 14:53:52 -0500 Subject: [PATCH 205/216] Update frontmatter author icons Merges https://github.com/greenelab/manubot-rootstock/pull/166 Improves formatting and centering of icons. Uses github and twitter icons from https://fontawesome.com --- content/images/github.svg | 8 +++----- content/images/orcid.svg | 14 +++----------- content/images/twitter.svg | 9 ++++----- 3 files changed, 10 insertions(+), 21 deletions(-) diff --git a/content/images/github.svg b/content/images/github.svg index ae3a1d0..e104aa7 100644 --- a/content/images/github.svg +++ b/content/images/github.svg @@ -1,6 +1,4 @@ - - - - GitHub icon - + + + diff --git a/content/images/orcid.svg b/content/images/orcid.svg index 5877e42..28b2328 100644 --- a/content/images/orcid.svg +++ b/content/images/orcid.svg @@ -1,12 +1,4 @@ - - - - - - - - - - - + + + diff --git a/content/images/twitter.svg b/content/images/twitter.svg index 92ea80c..dc8cd89 100644 --- a/content/images/twitter.svg +++ b/content/images/twitter.svg @@ -1,5 +1,4 @@ - - - - - + + + + From ec359f63c7d2b9ec0f2c04fabccd648f0b6a1ab9 Mon Sep 17 00:00:00 2001 From: Vincent Rubinetti Date: Tue, 29 Jan 2019 12:09:18 -0500 Subject: [PATCH 206/216] Explicit fill color in icon SVGs Merges https://github.com/greenelab/manubot-rootstock/pull/172 Fixes issue that fill="currentColor" did not work as expected because SVGs are loaded from URLs rather than defined inline. Hence, revert to specifying a specific color. --- content/images/github.svg | 2 +- content/images/orcid.svg | 2 +- content/images/twitter.svg | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/content/images/github.svg b/content/images/github.svg index e104aa7..54bbd93 100644 --- a/content/images/github.svg +++ b/content/images/github.svg @@ -1,4 +1,4 @@ - + diff --git a/content/images/orcid.svg b/content/images/orcid.svg index 28b2328..15d082c 100644 --- a/content/images/orcid.svg +++ b/content/images/orcid.svg @@ -1,4 +1,4 @@ - + diff --git a/content/images/twitter.svg b/content/images/twitter.svg index dc8cd89..1b590e2 100644 --- a/content/images/twitter.svg +++ b/content/images/twitter.svg @@ -1,4 +1,4 @@ - + From 658bcd763deae50732867a471d38760a90b13641 Mon Sep 17 00:00:00 2001 From: Vincent Rubinetti Date: Fri, 1 Feb 2019 17:48:56 -0500 Subject: [PATCH 207/216] HTML themes: rewrite default.html with new style Merges https://github.com/greenelab/manubot-rootstock/pull/171 Various display improvements, simplified code structure with modern best practices. --- build/themes/default.html | 1407 ++++++++++-------------------------- content/00.front-matter.md | 6 +- 2 files changed, 370 insertions(+), 1043 deletions(-) diff --git a/build/themes/default.html b/build/themes/default.html index 2bfcda0..77fde56 100644 --- a/build/themes/default.html +++ b/build/themes/default.html @@ -1,1093 +1,420 @@ + diff --git a/content/00.front-matter.md b/content/00.front-matter.md index f8d4e32..0f1b87a 100644 --- a/content/00.front-matter.md +++ b/content/00.front-matter.md @@ -27,15 +27,15 @@ on {{date}}. {% for author in authors %} + **{{author.name}}**
{%- if author.orcid is defined and author.orcid is not none %} - ![ORCID icon](images/orcid.svg){height="13px" width="13px"} + ![ORCID icon](images/orcid.svg){.inline_icon} [{{author.orcid}}](https://orcid.org/{{author.orcid}}) {%- endif %} {%- if author.github is defined and author.github is not none %} - · ![GitHub icon](images/github.svg){height="13px" width="13px"} + · ![GitHub icon](images/github.svg){.inline_icon} [{{author.github}}](https://github.com/{{author.github}}) {%- endif %} {%- if author.twitter is defined and author.twitter is not none %} - · ![Twitter icon](images/twitter.svg){height="13px" width="13px"} + · ![Twitter icon](images/twitter.svg){.inline_icon} [{{author.twitter}}](https://twitter.com/{{author.twitter}}) {%- endif %}
From f559600ff1965899b20874e71874794c05787087 Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Wed, 6 Feb 2019 11:24:17 -0500 Subject: [PATCH 208/216] shortDOI support & update environment on 2019-02-06 Merges https://github.com/greenelab/manubot-rootstock/pull/174 * Environment update on 2019-02-06 Does not upgrade weasyprint or cairo, due to the following error during WeasyPrint execution: OSError: dlopen() failed to load a library: cairo / cairo-2 / cairo-gobject-2 Specify cairocffi dependency in environment.yml, since v0.9 from PyPI seemed to trigger the OSError. * Add shortDOI support & usage Update repo to https://github.com/manubot/manubot greenelab/manubot relocated to manubot/manubot as per https://github.com/manubot/manubot/issues/94. 02.delete-me.md still references greenelab/manubot, however it is not worth creating merge conflicts to update just this. --- .travis.yml | 4 ++-- USAGE.md | 6 +++++- build/environment.yml | 9 +++++---- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 29420f4..f0d96b3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,15 +5,15 @@ branches: only: - master before_install: - - wget https://repo.continuum.io/miniconda/Miniconda3-4.5.11-Linux-x86_64.sh + - wget https://repo.continuum.io/miniconda/Miniconda3-4.5.12-Linux-x86_64.sh --output-document miniconda.sh - bash miniconda.sh -b -p $HOME/miniconda - source $HOME/miniconda/etc/profile.d/conda.sh - hash -r - conda config --set always_yes yes --set changeps1 no - - conda info --all install: - conda env create --quiet --file build/environment.yml + - conda list --name manubot - conda activate manubot script: - sh build/build.sh diff --git a/USAGE.md b/USAGE.md index 2379e7b..0b522b6 100644 --- a/USAGE.md +++ b/USAGE.md @@ -76,6 +76,10 @@ Therefore, citations must be of the following form: `@source:identifier`, where When choosing which source to use for a citation, we recommend the following order: 1. DOI (Digital Object Identifier), cite like `@doi:10.15363/thinklab.4`. + Shortened versions of DOIs can be created at [shortdoi.org](http://shortdoi.org/). + shortDOIs begin with `10/` rather than `10.` and can also be cited. + For example, Manubot will expand `@doi:10/993` to the DOI above. + We suggest using shortDOIs to cite DOIs containing forbidden characters, such as `(` or `)`. 2. PubMed Central ID, cite like `@pmcid:PMC4497619`. 3. PubMed ID, cite like `@pmid:26158728`. 4. _arXiv_ ID, cite like `@arxiv:1508.06576v2`. @@ -165,7 +169,7 @@ Note that `affiliations` should be a list to allow for multiple affiliations per ## Manubot feedback -If you experience any issues with the Manubot or would like to contribute to its source code, please visit [`greenelab/manubot`](https://github.com/greenelab/manubot) or [`greenelab/manubot-rootstock`](https://github.com/greenelab/manubot-rootstock). +If you experience any issues with the Manubot or would like to contribute to its source code, please visit [`manubot/manubot`](https://github.com/manubot/manubot) or [`greenelab/manubot-rootstock`](https://github.com/greenelab/manubot-rootstock). ## Examples diff --git a/build/environment.yml b/build/environment.yml index 60e6e73..aa43320 100644 --- a/build/environment.yml +++ b/build/environment.yml @@ -3,20 +3,21 @@ channels: - conda-forge dependencies: - conda-forge::cairo=1.14.12 + - conda-forge::cairocffi=0.8.0 - conda-forge::cffi=1.11.5 - conda-forge::ghp-import=0.5.5 - conda-forge::jinja2=2.10 - conda-forge::jsonschema=2.6.0 - - conda-forge::pandas=0.23.4 - - conda-forge::pandoc=2.5 + - conda-forge::pandas=0.24.0 + - conda-forge::pandoc=2.6 - conda-forge::pango=1.40.14 - - conda-forge::python=3.6.5 + - conda-forge::python=3.6.7 - conda-forge::pyyaml=3.13 - conda-forge::requests=2.21.0 - conda-forge::watchdog=0.8.3 - pip: - errorhandler==2.0.1 - - git+https://github.com/greenelab/manubot@552dd99aaa148365a29a87176fedd626d4642d3c + - git+https://github.com/manubot/manubot@4505db3fd8a9dc1ba26c36f3fea9accbf02d47d8 - jsonref==0.2 - opentimestamps-client==0.6.0 - opentimestamps==0.3.0 From 2a4c813179e0b4d51b8d4141a55684d206679e1d Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Tue, 12 Feb 2019 20:11:25 -0500 Subject: [PATCH 209/216] Relocate repository to manubot/rootstock Merges https://github.com/manubot/rootstock/pull/180 Closes https://github.com/greenelab/manubot-rootstock/issues/175 Closes https://github.com/greenelab/manubot-rootstock/pull/124 (supersedes) Relocate repository from https://github.com/greenelab/manubot-rootstock to https://github.com/manubot/rootstock. Relocate continuous integration from https://travis-ci.org/greenelab/manubot-rootstock to https://travis-ci.com/manubot/rootstock --- README.md | 22 +++++++++++----------- SETUP.md | 36 +++++++++++++++++++----------------- USAGE.md | 4 ++-- build/README.md | 2 +- build/assets/style.csl | 2 +- ci/README.md | 2 +- ci/deploy.key.enc | Bin 3248 -> 3248 bytes ci/deploy.key.pub | 2 +- ci/deploy.sh | 8 ++++---- 9 files changed, 40 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index 7b55c40..7e83764 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,9 @@ -[![HTML Manuscript](https://img.shields.io/badge/manuscript-HTML-blue.svg)](https://greenelab.github.io/manubot-rootstock/) -[![PDF Manuscript](https://img.shields.io/badge/manuscript-PDF-blue.svg)](https://greenelab.github.io/manubot-rootstock/manuscript.pdf) -[![Build Status](https://travis-ci.org/greenelab/manubot-rootstock.svg?branch=master)](https://travis-ci.org/greenelab/manubot-rootstock) +[![HTML Manuscript](https://img.shields.io/badge/manuscript-HTML-blue.svg)](https://manubot.github.io/rootstock/) +[![PDF Manuscript](https://img.shields.io/badge/manuscript-PDF-blue.svg)](https://manubot.github.io/rootstock/manuscript.pdf) +[![Build Status](https://travis-ci.com/manubot/rootstock.svg?branch=master)](https://travis-ci.com/manubot/rootstock) ## Manuscript description @@ -19,10 +19,10 @@ Actual manuscript instances will clone this repository (see [`SETUP.md`](SETUP.m Manubot is a system for writing scholarly manuscripts via GitHub. Manubot automates citations and references, versions manuscripts using git, and enables collaborative writing via GitHub. -The [Manubot Rootstock repository](https://git.io/vQSvo) is a general purpose template for creating new Manubot instances, as detailed in [`SETUP.md`](SETUP.md). +The [rootstock repository](https://git.io/fhQH1) is a general purpose template for creating new Manubot instances, as detailed in [`SETUP.md`](SETUP.md). See [`USAGE.md`](USAGE.md) for documentation how to write a manuscript. -Please open [an issue](https://github.com/greenelab/manubot-rootstock/issues) for questions related to Manubot usage, bug reports, or general inquiries. +Please open [an issue](https://git.io/fhQHM) for questions related to Manubot usage, bug reports, or general inquiries. ### Repository directories & files @@ -69,17 +69,17 @@ sh build/autobuild.sh ### Continuous Integration -[![Build Status](https://travis-ci.org/greenelab/manubot-rootstock.svg?branch=master)](https://travis-ci.org/greenelab/manubot-rootstock) +[![Build Status](https://travis-ci.com/manubot/rootstock.svg?branch=master)](https://travis-ci.com/manubot/rootstock) Whenever a pull request is opened, Travis CI will test whether the changes break the build process to generate a formatted manuscript. The build process aims to detect common errors, such as invalid citations. If your pull request build fails, see the Travis CI logs for the cause of failure and revise your pull request accordingly. -When a commit to the `master` branch occurs (for example, when a pull request is merged), Travis CI builds the manuscript and writes the results to the [`gh-pages`](https://github.com/greenelab/manubot-rootstock/tree/gh-pages) and [`output`](https://github.com/greenelab/manubot-rootstock/tree/output) branches. +When a commit to the `master` branch occurs (for example, when a pull request is merged), Travis CI builds the manuscript and writes the results to the [`gh-pages`](https://github.com/manubot/rootstock/tree/gh-pages) and [`output`](https://github.com/manubot/rootstock/tree/output) branches. The `gh-pages` branch uses [GitHub Pages](https://pages.github.com/) to host the following URLs: -+ **HTML manuscript** at https://greenelab.github.io/manubot-rootstock/ -+ **PDF manuscript** at https://greenelab.github.io/manubot-rootstock/manuscript.pdf ++ **HTML manuscript** at https://manubot.github.io/rootstock/ ++ **PDF manuscript** at https://manubot.github.io/rootstock/manuscript.pdf For continuous integration configuration details, see [`.travis.yml`](.travis.yml). @@ -94,7 +94,7 @@ We encourage users to openly license their manuscripts, which is the default as [![License: CC0 1.0](https://img.shields.io/badge/License%20Parts-CC0%201.0-lightgrey.svg)](https://creativecommons.org/publicdomain/zero/1.0/) Except when noted otherwise, the entirety of this repository is licensed under a CC BY 4.0 License ([`LICENSE.md`](LICENSE.md)), which allows reuse with attribution. -Please attribute by linking to https://github.com/greenelab/manubot-rootstock. +Please attribute by linking to https://github.com/manubot/rootstock. Since CC BY is not ideal for code and data, certain repository components are also released under the CC0 1.0 public domain dedication ([`LICENSE-CC0.md`](LICENSE-CC0.md)). All files matched by the following glob patterns are dual licensed under CC BY 4.0 and CC0 1.0: @@ -118,4 +118,4 @@ Except for the following files with different licenses: + `build/assets/anchors.js` which is [released](https://www.bryanbraun.com/anchorjs/) under an [MIT License](https://opensource.org/licenses/MIT) -Please open [an issue](https://github.com/greenelab/manubot-rootstock/issues) for any question related to licensing. +Please open [an issue](https://github.com/manubot/rootstock/issues) for any question related to licensing. diff --git a/SETUP.md b/SETUP.md index a32ae13..4f59302 100644 --- a/SETUP.md +++ b/SETUP.md @@ -1,9 +1,9 @@ -# Cloning manubot-rootstock to create a new manuscript +# Cloning the manubot/rootstock repository to create a new manuscript The process to create a new Manubot manuscript is a bit challenging, because it requires a few steps that are difficult to automate. However, you will only have to perform these steps once for each manuscript. These steps should be performed in a terminal, starting in the directory where you want the manuscript folder be created. -Setup is supported on Linux and macOS, but [**not on Windows**](https://github.com/greenelab/manubot-rootstock/issues/91). +Setup is supported on Linux and macOS, but [**not on Windows**](https://github.com/manubot/rootstock/issues/91). ## Configuration @@ -14,10 +14,10 @@ In general, assume that all commands in this setup are case-sensitive. **Edit the following commands with your manuscript's information:** ```sh -# GitHub username (change from greenelab) -OWNER=greenelab -# Repository name (change from manubot-rootstock) -REPO=manubot-rootstock +# GitHub username (change from manubot) +OWNER=manubot +# Repository name (change from rootstock) +REPO=rootstock ``` ## Create repository @@ -25,15 +25,15 @@ REPO=manubot-rootstock **Execute the remaining commands verbatim.** They do not need to be edited (if the setup works as intended). -Next you must clone `greenelab/manubot-rootstock` and configure its branches and remotes: +Next you must clone `manubot/rootstock` and configure its branches and remotes: ```sh -# Clone greenelab/manubot-rootstock -git clone https://github.com/greenelab/manubot-rootstock.git $REPO +# Clone manubot/rootstock +git clone https://github.com/manubot/rootstock.git $REPO cd $REPO # Configure remotes and branches -git remote add rootstock https://github.com/greenelab/manubot-rootstock.git +git remote add rootstock https://github.com/manubot/rootstock.git git checkout --orphan gh-pages git rm -r --force . git commit --allow-empty \ @@ -61,7 +61,7 @@ git push --set-upstream origin output ## Continuous integration -Now you must manually enable Travis CI for the new repository at https://travis-ci.org. +Now you must manually enable Travis CI for the new repository at https://travis-ci.com. Click the `+` sign to "Add New Repository". If you don't see your repository listed, push the "Sync account" button. Finally, flick the repository's switch to enable CI. @@ -89,10 +89,11 @@ Give the key a descriptive title, such as "Travis CI Manubot". For the next step, you need the [Travis command line client](https://github.com/travis-ci/travis.rb) installed. This program is a Ruby gem: install it with `gem install travis` (not `apt install travis`, which is a different program). -After the install, you will need to provide your credentials to login to travis with `travis login --org`. +After the install, you will need to provide your credentials to login to travis with `travis login`. ```sh travis encrypt-file \ + --com \ --repo=$OWNER/$REPO \ --force \ deploy.key > travis-encrypt-file.log @@ -119,6 +120,7 @@ Next, limit [concurrent](https://blog.travis-ci.com/2014-07-18-per-repository-co ```sh travis settings \ + --com \ --repo=$OWNER/$REPO \ maximum_number_of_builds --set 1 ``` @@ -141,8 +143,8 @@ Now update `README.md` files to reference the new repository: ```sh # Perform substitutions -sed "s/greenelab/$OWNER/g" README.md > tmp && mv -f tmp README.md -sed "s/manubot-rootstock/$REPO/g" README.md > tmp && mv -f tmp README.md +sed "s/manubot\/rootstock/$OWNER\/$REPO/g" README.md > tmp && mv -f tmp README.md +sed "s/manubot\.github\.io\/rootstock/$OWNER\.github\.io\/$REPO/g" README.md > tmp && mv -f tmp README.md # Remove deletable content file git rm content/02.delete-me.md @@ -168,9 +170,9 @@ git push origin master You should be good to go now. A good first step is to modify `content/metadata.yaml` with the relevant information for your manuscript. -# Merging upstream manubot-rootstock changes +# Merging upstream rootstock changes -This section will describe how to incorporate changes to manubot-rootstock that occurred since initializing your manuscript. +This section will describe how to incorporate changes to rootstock that occurred since initializing your manuscript. You will want to do this if there are new enhancements or bugfixes that you want to incorporate. This process can be difficult, especially if conflicts have arisen, and is recommended only for advanced git users. @@ -182,7 +184,7 @@ First, checkout a new branch to use as the pull request head branch: git checkout -b rootstock-`date '+%Y-%m-%d'` ``` -Second, pull the new commits from manubot-rootstock, but do not automerge: +Second, pull the new commits from rootstock, but do not automerge: ```sh git pull --no-ff --no-commit rootstock master diff --git a/USAGE.md b/USAGE.md index 0b522b6..4447603 100644 --- a/USAGE.md +++ b/USAGE.md @@ -1,6 +1,6 @@ # Manubot usage guidelines -This repository uses [Manubot](https://github.com/greenelab/manubot-rootstock) to automatically produce a manuscript from the source in the [`content`](content) directory. +This repository uses [Manubot](https://manubot.org) to automatically produce a manuscript from the source in the [`content`](content) directory. ## Manubot markdown @@ -169,7 +169,7 @@ Note that `affiliations` should be a list to allow for multiple affiliations per ## Manubot feedback -If you experience any issues with the Manubot or would like to contribute to its source code, please visit [`manubot/manubot`](https://github.com/manubot/manubot) or [`greenelab/manubot-rootstock`](https://github.com/greenelab/manubot-rootstock). +If you experience any issues with the Manubot or would like to contribute to its source code, please visit [`manubot/manubot`](https://github.com/manubot/manubot) or [`manubot/rootstock`](https://github.com/manubot/rootstock). ## Examples diff --git a/build/README.md b/build/README.md index e79063b..3be7e94 100644 --- a/build/README.md +++ b/build/README.md @@ -11,7 +11,7 @@ For example, use the command `BUILD_DOCX=true sh build/build.sh`. To export DOCX for all Travis builds, set a [Travis environment variable](https://docs.travis-ci.com/user/environment-variables/#Defining-Variables-in-Repository-Settings). Currently, equation numbers via `pandoc-eqnos` are not supported for DOCX output. There is varying support for embedding images in DOCX output. -Please reference [Pull Request #40](https://github.com/greenelab/manubot-rootstock/pull/40) for possible solutions and continued discussion. +Please reference [Pull Request #40](https://github.com/manubot/rootstock/pull/40) for possible solutions and continued discussion. ## Environment diff --git a/build/assets/style.csl b/build/assets/style.csl index 07ba262..51b1f49 100644 --- a/build/assets/style.csl +++ b/build/assets/style.csl @@ -5,7 +5,7 @@ Manubot http://www.zotero.org/styles/manubot - + Daniel Himmelstein daniel.himmelstein@gmail.com diff --git a/ci/README.md b/ci/README.md index a8e0d1a..abfd2ed 100644 --- a/ci/README.md +++ b/ci/README.md @@ -1,6 +1,6 @@ # Continuous integration / analysis directory -[![Build Status](https://travis-ci.org/greenelab/manubot-rootstock.svg?branch=master)](https://travis-ci.org/greenelab/manubot-rootstock) +[![Build Status](https://travis-ci.com/manubot/rootstock.svg?branch=master)](https://travis-ci.com/manubot/rootstock) This repository uses [continuous analysis](https://doi.org/10.1101/056473 "Reproducible Computational Workflows with Continuous Analysis") to create the manuscript and commit it back to GitHub. [`deploy.sh`](deploy.sh) runs on successful `master` branch builds that are not pull requests. diff --git a/ci/deploy.key.enc b/ci/deploy.key.enc index 28adfa66e36e6801295509a555478e06e006332f..6518ffd1257daef81a40e890671bed46d9f2c8eb 100644 GIT binary patch literal 3248 zcmV;h3{Uf&qPa3&?mNq1IY>b20)tJ>;o?p%EU_<)8J5SmBfi+3b91D3Xya%cn1^M( zakF~JinPeESP$Zw}|0AQG#NKS$5yG4m%hfkfw1444 zimvot`3(cv9Oj`2@R?Z1q_$xpeP~iI8n>9dW_4t^Ej~wq7hf$A*b*nl_ux47$MA}j zjrZyR{CJ}E?g5yo!u2EY+*jr&DNO=|A)#Q;wDP2gX}f{ zQV1Ru;V$ZZP(@&aD_UfC)O47X)WOWcx!(!B*hs3qdzJ{|6MdxQ3AfZx-Z!!(U}Kz) zAq&h`wq3bI{7mxxq^|6mF85m1U}TX`<=+U=6k^E2UCpmuSelVDSnud6sWcx^`CWrz zND8sJ%maSeq>-u4K5Yqq{!wX5?qnWM+KqzP26{uxo8IK%-$My8aBa0m`qmku1?2KF4o@tehTVHsy0K$VhT_*7A zzDDIg13jvF$D@HMv;kFlaYp&UE5qN5xSJ!Gn~{rcS)KZPioH6PuKc|Xd@%Fd9ZGo9 zA)%%i6xLr&kt7a;dnaY`A{zOtS(A{dmN~LEriv{xS78KYQR^v8d)vdX?l1kQ{S`>P zRgKf1F9Sj&TM5jt_$a7=aDIyty*o5AcuDZbxi|9({f_X2c3l)nb3 zem;d7Gf{yl^6_rRS&O~h1*m`lx=xc2h(iQV>POSsHayNFXdgIqDP1=Pj@o!Ro%FCf z7_)y;(EjeF(1mUY5=NpA^NjPW@eS+X$`H_-*0Q3+XP26k?3WyW{8FrJK~tK?3$d1d zEm00*(L2??*FEmU=Nm zfh*zLT?`PIJl5K!v!8E7Yuk<;9y=@mZrrnbE%(9pS1N|EZ#t8C3w!lT-|)&Va~wL5 z1Z-Ky8+AdN$1(!a$4+>BzxZ$@zuRJ7(5ZSu5$OoYX+d5~o+)hqGO#<2{V=+#1PK$d;4NN zQ}5bMTlzxC;`!J^Hc5Z5e)xN#ejCpMI^7~VweH<)&6(!g5&MSJ;}O>v`?vr9{twg= zx8K)(L_)P9$@$-u7n7OR)*Ev;=^V6-tK}weHhD99z&$}D5Q$RW~M0*uAjamRpQY8*=hC1gfph?{`^l8_Bu6?rN>`P zG@xv8o@#mWVMRLOF0?d(^{d01Zcd2~QxCI*G~D9Y)Hh|e93$D;=m}}aJy-ypSm6X1 z`0%EN8QFQM@%qfoZRARnJ{pjG$sM0h?T%p?fC>z!4RbOfyiJMyyVU!qCVppo=Dy6# z;KhK8SaYS^=yRSuNha*1n03iDO6)bH@{7Kq^14+HMZk8O&8X~5I^>`G^B_yU~sT2Sgu*()8-TUpagVwr}Gf6SjQUz$9 zrh^5o9tMdkVK0dZPI1g7am|q&7_$vdW6@G(mhNKGPV+6?5Bkd8T!J6S=5be0t`^o1 z;_Wv+IG{elqH}RD^kDalR_cz0mpGe{Ln@g6^97-v z*G=;{a!(Y@a^AAwO6*fxA8hP9zx}mMLbm5nj|x}m4+gpK)i6tAb#(@63k$yWFQb1G zE=>pY(t$|0X(iDv5lTWLC_mm({e_~VF6-(i);eF{D@L0Dgo!MOFR{vMh2lnisR5qQ zRr%a^0!+Qg0LE%N25@XJ5~Nfs^jS8AyE=GuVKNe6y^HzE0H#H_f<%8kH;05`fHcq! zl?5+mkxQjYQ2%06VW4t`edb?%V32?5jkaJ`16pEiMPffF0pz>r2j>(s&0S7PDD1MS z3EV<$;NQ-`C|i9qVQojR7PlUm)TyYuXeU-O=H^+_AoN~%87?SiJe~*scHGlniYF5` z)Ff*AkKk*ZTO%^kGvcDOSWtY*4-gVSwXMTGk~|4L(?&%rk6CRGPu7Z_`&Tx4euA*0 zDqdPG@AOh;QY@7bpkJOZ_OfB7SwzgA(5I8$ol^p=sTWLCEo#0g<1T?g;nVal;5DEl ziBL_@rLlFi-Q$%{N)bV#P-XJ+46#2BXbbsUvysDb$eMIcB1u$43a;Wuajz?pM!<)+ z#Y1fn!bBh{)}3MSqGc!l(x|R6oVg*jyQ|*Jm6suyWU|e}JL@<0=*aY*?Tu23S`k;L zE7kg|f8z6kj#ojpy#6eL^$=n>aoq#Bemw|tE$UPn`0aWwLA~>^>K;`%p+Mtt^&*0N zCJojbFrN7CXKIV{W|>>`z>JmPSCs+tduZr*itrhoV8HVDcHIN%alRvmqaZ9sVJW`A?z0r5~!%FOlJeq z8XfBCBWspt)(kZU-w^Js7H$F@U#p9EfDJ`C($eIGi9F(=e?Na4U7Tiamd6#UYYNtd zyt-5R-XAI;5J!W{=C9snvD76ii0$l35L-|moDIxaMM&xYCPS=_mQ5Fp5CnT_*9BfD z^bBgeo=#R`8(Ubz&yk7Y-|8wTMbm!+J$EdJB&M^IdL;i5bcqRZ!FR=`R7@MJ&-c_) zSE8c7ZX^Y-dT)~%QdLuO`ZX#lJC2xNls#?_A?=zM7f8#Y-QcFVZ`<(L#2eu@!R?7L zg#{DFC~rnd2Ug&V*w~1X@3HPE&A-q|c`9>lkpM1+!cYv29;UyY;WE5cXh6!>wm9P9g_`)% zM1eTKN`nP-`@g5Y-?!VYbt%-0P08mJkqBX^SIz@&oJ8SG$3tQfw~GZxaqX7tIX(az!15T?|W4r_CO$``uQU7YcV`#Hf#C-LlV9 zms_4%T`&XTxzNm0I>s3_ovHpv1Cn7n6?dyp55bK(a76`W zD{;?AFd!yBKC+@VM4v#O4gGaV{yVNwJ%we{Q%6$zEvImsh9M}X@{<;7Ra%38bKN|S z_Z=CFKliWJetO-!3#MqbyF2hTIZcOskbkWU=`x8}7b4@xGZlB`a_7;f*x;Kxpo%Zb zaM87X6*&MSN;5>J>&^R4kTTSG8Dm+85iOL>9?CV``0z%kBG_U^FdwaGr|$`9k-hNa1)7H9VB-9Fb@iQ{PC){jSn1{mojR^X(cW3;>r zr1$Z=R8gzT8*9!1r$QELXIdys5Q8~;t%OjKp5yjKQov$gDBA&MzZZ4jqVFtRf&xX_ i#odRKROUlX><#nq@3CdKM4_rKw$Imtb>MSX2VmBbq8J@8qPjmgJQv6_8Xd!n?QnJ!q$~*ZMzt7N ze49>`a#19UY0AuIv-j}9<>#^VEr5kFF;9j^zx9^`*Jvo+pM(SU9yh!!u)1 zy17{oU!6wT$S8s5sxZU?;ZY?lZrwO%R9?u-q=RHu=~^FtJN?!{-&dqgNQdOebwMAf z|Ff&XXL3NAR_xIjtXWurbOHd-6@REVl#sFW<4;l_`hdq~*)?E|`$YCuPk2_vKoX|? zD~S~@l;Kwj^_^_i{e%jryX^*LI79SJ)9;$2+&lfKC1pvUftSx|m(MV0^4f|M%zApU zy_stp71}_{RU?&B+OS+eH=c-HiUYvON%%^V01=D{U+LEQlj+NOA_*wWBhSOJp~N&i ztd~in5pKHY2qCt6YLBw4_RQ=HWr33Wq-(0ZPkmeR-3489T}`S(t~V}BdH!;Y*V3ej z(HBH*M0$!nMRB6mrA;lk5ERTA#&3W=U4@%*Rh1Nq0~7c~l3y{uerI-wJ}kWRG(hbKJKNL3 zNwh5%VR?)RJ=)I**@7lXbheA1h!dXj9r3?Ne|Zk~bzv_ezNDU%VpD9)WSknpd-;mW z9ou`DmE#Bz@{jO8qb$7=j|JQ!ea0K`!8Xx*(?qaxi%!o6Qk$~WV|D`RMTKAI6VXqq zuxAxinja@R4u} zn=xL>p|@Hf#ez7F@d@jedz}?hcCpZ$PA2-3-_zNkVnWK_hm#5ai zN&`dCtHiQQD(6ilF#1k!$QON_$_xPJyq-0of(IMbDb}$>M{rlS#y7b&B0C;7WitgB z7PyF2$+2c|c~#wYx5x_W`IKiM?Z7DBNC7;oJIPXmkr?#pdUQSew1_*uaY#`)a6^=7 zjbzAh>ys(s=|*uomEVq<*gA3e*M{I1t|dXuNM6Eq8InTIw%ZL2sQrZlK&bFSlO7K9 zx_rVF8~nLwtHDC5I}K94PUF5aXdQj%YDYb-pCihKj-s;8x8fSgD124^FM8FU)ff(e zbiB|U+IWNBOCl1HSwtRe&iaP5GPhUWq0Wt468oXUZqx+Zpiwq&U+F*QlvJ5FnK@B) z{;+}Sl^P1@2`g5xG{b_fu*(albqQ=C@Ex|jvmVws(BO-6srpDrbr$&}^JNG~$GtOTthcM{C!LxR=77!FPV4Cv`^1jP_YRWi$S`Y9K&ur3n$!F`usFbHuk|Z!pv;2!lcS?f zg;KWb$BXCvM+FoVkEA>J6r}VB1a$RKj@h26TZ_7PPm)l$YOK9jt6nfa zZjRzOd_3`!er?vB&L;K3TAqP|OJ!$P&E=M^<3`vgD5w16P#Dy(XHkNVCBv8~x-bw> zckf`SV8b#vou=Fv?t~)~%5Q7L=ezaOAMr3P4^;V^z?G_|tWKkV!tNKkO0Fw|M{8iq z>h(V#%mnAQ!Zktn~nHGi27R zw)Yt&o&zh{bfcX^XV{g0u8kb8yI%TW-+VsnBj-hAvC_@J_DhrrQX!B{AaATzEeh~g zwNn8DOwrqx`6ySZZY@N1vUa zoW3s%lqK260)uot>qVrs@aYO}{m(v7maDgfomKLGXDJjGx#m?2b%NpgV2)T!{vqBA zW-%bK#S-rA%HBmrCTzJSYUiahb$camOGxzRFx7l=4GU=RA#0MYY#h4ul_w1=^BMJJ zYIL)-=wWshNgQ8(`s}lfNGn7}FwmbCIYWf)ebK|M$Cqabc~Og{{j!YSGzvB8KsQ|IKAn^ zW24wG2RI1`rOSiJ=F*HpYCMPd<2v_7N~jvAi+l20(nc7QRV~L*)D*@7qvUagM1q~= z8s$_=M9R~1JWfou3{gY#n`^NRx@GcGejymnLO#LOo9J(Fpm++a;4(ie>eeqYbnByb zDM>43^V|0eTLPo5w(0F7W`*`?#Pmf0j~)*H#V*fd^%mt95=;w!Yxu8`t(Zme9kWti z2SJ~8iimQ<-m&ONwzK49-pl^;Hm;?MFbwuV>1t3|0uCB<3)q}wYkP-ehZQ039rffP zBh(MVQP#ZP6B~j#j7`|qaafXwmB=>*YS^Rbi}aeV4mVEr_@y-t%ZG58JL)bL6%bcn z&z<;!_;?f`t<1I_>}kLyk`X+Id3otR+4o09*x?DRV8h-af;6u!+G!t2S!a>;mbB){ zThU1BU_EAR6shM0xVcC5PZrE84G^EM1U&8ZxFe$_`iTwk^!;C-AwC){hkt^t37NK@ z*|6jlKt;s?c!@eZI^iTmDo=-!x=+xP;|r#tyH#odJQ(Aw#?~>4_1p4Zz!I&(lNgq_ zcwM%WyxgLpKyWVLe9YghyKgA@!v*4B|8d?QsO0+2;gv<_Wj-x-`Zu#T1w%A{KbW$t zEHFeu=0qc#C3)L@K(QdcU>={ZXY6bGs}T;<-nAfG`H^%PKTI$nchWFA3rc}5FnYs;GJwnTu7G0(G@?n)#nuGh^@vT6dE zJt2=>yS({>zqVgTZj7>}w&uNd2dOAT*=e63daLyi=zDf;A|JFDJ6W^?@k6+ZH?GNh zFF*ryc6S}LV!j{;$j^+XFr!fNEen`~Zih8!AH9?3wU(eQWH&y>R<#?OK80BS{q6?C zQA1U|kBpL$YG;ST%qWJTQ;TQnWDF|(BH6Bx@+GtXxTYx+KqHBS9300by;{ Date: Thu, 28 Feb 2019 01:16:32 -0800 Subject: [PATCH 210/216] Fix SETUP.md random string & psutil install Merges https://github.com/manubot/rootstock/pull/182 Update SETUP.md to replace the correct random string Fix missing x86_64-conda_cos6-linux-gnu-gcc in psutil install by installing psutil using conda. .travis.yml: one configuration option per line (non-functional whitespace change) --- .travis.yml | 4 +++- SETUP.md | 2 +- build/environment.yml | 1 + 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index f0d96b3..7f4cd22 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,9 @@ before_install: - bash miniconda.sh -b -p $HOME/miniconda - source $HOME/miniconda/etc/profile.d/conda.sh - hash -r - - conda config --set always_yes yes --set changeps1 no + - conda config + --set always_yes yes + --set changeps1 no install: - conda env create --quiet --file build/environment.yml - conda list --name manubot diff --git a/SETUP.md b/SETUP.md index 4f59302..5d366f2 100644 --- a/SETUP.md +++ b/SETUP.md @@ -113,7 +113,7 @@ The following command automates this substitution. TRAVIS_ENCRYPT_ID=`python -c "import re; \ text = open('travis-encrypt-file.log').read(); \ print(re.search('encrypted_([a-f0-9]+)_key', text).group(1))"` -sed "s/f2f00aaf6402/$TRAVIS_ENCRYPT_ID/g" deploy.sh > tmp && mv -f tmp deploy.sh +sed "s/9befd6eddffe/$TRAVIS_ENCRYPT_ID/g" deploy.sh > tmp && mv -f tmp deploy.sh ``` Next, limit [concurrent](https://blog.travis-ci.com/2014-07-18-per-repository-concurrency-setting/) Travis CI jobs to ensure previous builds deploy before subsequent ones begin: diff --git a/build/environment.yml b/build/environment.yml index aa43320..553a5db 100644 --- a/build/environment.yml +++ b/build/environment.yml @@ -11,6 +11,7 @@ dependencies: - conda-forge::pandas=0.24.0 - conda-forge::pandoc=2.6 - conda-forge::pango=1.40.14 + - conda-forge::psutil=5.5.1 - conda-forge::python=3.6.7 - conda-forge::pyyaml=3.13 - conda-forge::requests=2.21.0 From 720febe2e48805035e4b5c7862c63b353e3cf65e Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Sat, 2 Mar 2019 07:37:29 -0500 Subject: [PATCH 211/216] CI: only deploy on push TRAVIS_EVENT_TYPE Merges https://github.com/manubot/rootstock/pull/184 Previously, ci/deploy.sh could be run for push, api, or cron builds, but not for pull_request builds. This commit further restricts when deployment is run to just push builds. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 7f4cd22..34b14d8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,5 +24,5 @@ cache: - ci/cache after_success: - test $TRAVIS_BRANCH = "master" && - test $TRAVIS_PULL_REQUEST = "false" && + test $TRAVIS_EVENT_TYPE = "push" && sh ci/deploy.sh From e5ab24a6e02728273b470e7adf46eeb20aebb4d2 Mon Sep 17 00:00:00 2001 From: Vincent Rubinetti Date: Fri, 8 Mar 2019 18:05:00 -0500 Subject: [PATCH 212/216] HTML: default theme updates Merges https://github.com/manubot/rootstock/pull/178 Various aesthetic updates, refactoring, edits for plugins, and additional comments --- build/themes/default.html | 365 +++++++++++++++++++++++++------------- 1 file changed, 245 insertions(+), 120 deletions(-) diff --git a/build/themes/default.html b/build/themes/default.html index 77fde56..9d69734 100644 --- a/build/themes/default.html +++ b/build/themes/default.html @@ -1,6 +1,6 @@ - From 6303fcfbac952c2fab8b6fff2cf82f18949fd9bb Mon Sep 17 00:00:00 2001 From: Anthony Gitter Date: Tue, 12 Mar 2019 08:28:23 -0500 Subject: [PATCH 213/216] Update rootstock manual JSON CSL in example reference Merges https://github.com/manubot/rootstock/pull/186 --- content/manual-references.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/content/manual-references.json b/content/manual-references.json index be37a81..dda7f5b 100644 --- a/content/manual-references.json +++ b/content/manual-references.json @@ -1,14 +1,14 @@ [ { "type": "webpage", - "standard_citation": "url:https://github.com/greenelab/manubot-rootstock", - "URL": "https://github.com/greenelab/manubot-rootstock", - "title": "greenelab/manubot-rootstock GitHub repository", + "standard_citation": "url:https://github.com/manubot/rootstock", + "URL": "https://github.com/manubot/rootstock", + "title": "manubot/rootstock GitHub repository", "container-title": "GitHub", "issued": { "date-parts": [ [ - 2017 + 2019 ] ] }, @@ -19,4 +19,4 @@ } ] } -] \ No newline at end of file +] From af1d47a0ec5f33d8fc99deab2ac23b697983b673 Mon Sep 17 00:00:00 2001 From: Vincent Rubinetti Date: Tue, 12 Mar 2019 10:21:45 -0400 Subject: [PATCH 214/216] HTML: add modular plugins for webpage interactivity Merges https://github.com/manubot/rootstock/pull/179 Closes https://github.com/manubot/rootstock/issues/185 Improves https://github.com/manubot/rootstock/issues/165 Adds plugins that can be included in the pandoc build commands to enable various interactive frontend features. Plugins are located in `build/plugins/` and consist of: - accordion (new) - analytics (modified) - anchors (replaced) - hypothesis (modified) - jump-to-first (new) - lightbox (new) - link-highlight (new) - math (modifies how MathJax is loaded) - table-of-contents (new) - table-scroll (new) - tooltips (new) --- README.md | 4 - build/build.sh | 13 +- build/plugins/accordion.html | 273 ++++++++++++ build/plugins/analytics.html | 7 +- build/plugins/anchors.html | 159 ++++++- build/plugins/hypothesis.html | 31 +- build/plugins/jump-to-first.html | 273 ++++++++++++ build/plugins/lightbox.html | 592 +++++++++++++++++++++++++ build/plugins/link-highlight.html | 192 ++++++++ build/plugins/math.html | 22 + build/plugins/table-of-contents.html | 282 ++++++++++++ build/plugins/table-scroll.html | 68 +++ build/plugins/tooltips.html | 627 +++++++++++++++++++++++++++ build/themes/default.html | 549 +++++++++++++++++++++++ 14 files changed, 3067 insertions(+), 25 deletions(-) create mode 100644 build/plugins/accordion.html create mode 100644 build/plugins/jump-to-first.html create mode 100644 build/plugins/lightbox.html create mode 100644 build/plugins/link-highlight.html create mode 100644 build/plugins/math.html create mode 100644 build/plugins/table-of-contents.html create mode 100644 build/plugins/table-scroll.html create mode 100644 build/plugins/tooltips.html diff --git a/README.md b/README.md index 7e83764..dcd0975 100644 --- a/README.md +++ b/README.md @@ -114,8 +114,4 @@ All other files are only available under CC BY 4.0, including: + `*.pdf` + `*.docx` -Except for the following files with different licenses: - -+ `build/assets/anchors.js` which is [released](https://www.bryanbraun.com/anchorjs/) under an [MIT License](https://opensource.org/licenses/MIT) - Please open [an issue](https://github.com/manubot/rootstock/issues) for any question related to licensing. diff --git a/build/build.sh b/build/build.sh index 69f6735..01f9284 100644 --- a/build/build.sh +++ b/build/build.sh @@ -33,11 +33,20 @@ pandoc --verbose \ --bibliography=$BIBLIOGRAPHY_PATH \ --csl=$CSL_PATH \ --metadata link-citations=true \ - --mathjax \ --include-after-body=build/themes/default.html \ - --include-after-body=build/plugins/analytics.html \ + --include-after-body=build/plugins/table-scroll.html \ --include-after-body=build/plugins/anchors.html \ + --include-after-body=build/plugins/accordion.html \ + --include-after-body=build/plugins/tooltips.html \ + --include-after-body=build/plugins/jump-to-first.html \ + --include-after-body=build/plugins/link-highlight.html \ + --include-after-body=build/plugins/table-of-contents.html \ + --include-after-body=build/plugins/lightbox.html \ + --mathjax \ + --variable math="" \ + --include-after-body=build/plugins/math.html \ --include-after-body=build/plugins/hypothesis.html \ + --include-after-body=build/plugins/analytics.html \ --output=output/manuscript.html \ $INPUT_PATH diff --git a/build/plugins/accordion.html b/build/plugins/accordion.html new file mode 100644 index 0000000..c47ae45 --- /dev/null +++ b/build/plugins/accordion.html @@ -0,0 +1,273 @@ + + + + + + + diff --git a/build/plugins/analytics.html b/build/plugins/analytics.html index 5264bbd..0e641a1 100644 --- a/build/plugins/analytics.html +++ b/build/plugins/analytics.html @@ -1,4 +1,3 @@ - - - \ No newline at end of file + + + diff --git a/build/plugins/anchors.html b/build/plugins/anchors.html index 5f4777b..92015d1 100644 --- a/build/plugins/anchors.html +++ b/build/plugins/anchors.html @@ -1,15 +1,148 @@ + + + + + + diff --git a/build/plugins/hypothesis.html b/build/plugins/hypothesis.html index bbc90f7..b38888e 100644 --- a/build/plugins/hypothesis.html +++ b/build/plugins/hypothesis.html @@ -1,2 +1,29 @@ - - + + + + + + + diff --git a/build/plugins/jump-to-first.html b/build/plugins/jump-to-first.html new file mode 100644 index 0000000..487ddae --- /dev/null +++ b/build/plugins/jump-to-first.html @@ -0,0 +1,273 @@ + + + + + + + diff --git a/build/plugins/lightbox.html b/build/plugins/lightbox.html new file mode 100644 index 0000000..ec90566 --- /dev/null +++ b/build/plugins/lightbox.html @@ -0,0 +1,592 @@ + + + + + + + + + + + diff --git a/build/plugins/link-highlight.html b/build/plugins/link-highlight.html new file mode 100644 index 0000000..f9a8469 --- /dev/null +++ b/build/plugins/link-highlight.html @@ -0,0 +1,192 @@ + + + diff --git a/build/plugins/math.html b/build/plugins/math.html new file mode 100644 index 0000000..f1a3861 --- /dev/null +++ b/build/plugins/math.html @@ -0,0 +1,22 @@ + + + + + + + diff --git a/build/plugins/table-of-contents.html b/build/plugins/table-of-contents.html new file mode 100644 index 0000000..d33d6e2 --- /dev/null +++ b/build/plugins/table-of-contents.html @@ -0,0 +1,282 @@ + + + + + + + diff --git a/build/plugins/table-scroll.html b/build/plugins/table-scroll.html new file mode 100644 index 0000000..2a4473a --- /dev/null +++ b/build/plugins/table-scroll.html @@ -0,0 +1,68 @@ + + + diff --git a/build/plugins/tooltips.html b/build/plugins/tooltips.html new file mode 100644 index 0000000..82390b6 --- /dev/null +++ b/build/plugins/tooltips.html @@ -0,0 +1,627 @@ + + + + + + + + + + + diff --git a/build/themes/default.html b/build/themes/default.html index 9d69734..b65d6fe 100644 --- a/build/themes/default.html +++ b/build/themes/default.html @@ -542,4 +542,553 @@ /* nudge to make room for equation auto-number */ margin-right: 40px !important; } + + /* -------------------------------------------------- */ + /* table scroll plugin */ + /* -------------------------------------------------- */ + + @media only screen { + /* table wrapper */ + .table_wrapper { + /* show scrollbar on tables if necessary to prevent overflow */ + overflow: auto; + width: 100%; + margin: 20px 0; + } + + /* table within table wrapper */ + .table_wrapper table, + .table_wrapper table * { + /* don't break table words */ + word-break: normal !important; + } + + .table_wrapper > table { + /* move margins from table to table_wrapper to allow margin collapsing */ + margin: 0; + } + } + + /* -------------------------------------------------- */ + /* anchors plugin */ + /* -------------------------------------------------- */ + + @media only screen { + /* anchor button */ + .anchor { + opacity: 0; + margin-left: 5px; + } + + /* anchor buttons within

's */ + h2 .anchor { + margin-left: 10px; + } + + /* anchor buttons when hovered/focused and anything containing an anchor button when hovered */ + *:hover > .anchor, + .anchor:hover, + .anchor:focus { + opacity: 1; + } + + /* anchor button when hovered */ + .anchor:hover { + cursor: pointer; + } + } + + /* always show anchor button on devices with no mouse/hover ability */ + @media (hover: none) { + .anchor { + opacity: 1; + } + } + + /* always hide anchor button on print */ + @media only print { + .anchor { + display: none; + } + } + + /* -------------------------------------------------- */ + /* accordion plugin */ + /* -------------------------------------------------- */ + + @media only screen { + /* accordion arrow button */ + .accordion_arrow { + margin-right: 10px; + } + + /* arrow icon when

data-collapsed attribute true */ + h2[data-collapsed="true"] > .accordion_arrow > svg { + transform: rotate(-90deg); + } + + /* all elements (except

's) when data-collapsed attribute true */ + *:not(h2)[data-collapsed="true"] { + display: none; + } + + /* accordion arrow button when hovered and

's when hovered */ + .accordion_arrow:hover, + h2[data-collapsed="true"]:hover, + h2[data-collapsed="false"]:hover { + cursor: pointer; + } + } + + /* always hide accordion arrow button on print */ + @media only print { + .accordion_arrow { + display: none; + } + } + + /* -------------------------------------------------- */ + /* tooltips plugin */ + /* -------------------------------------------------- */ + + @media only screen { + /* tooltip container */ + #tooltip { + position: absolute; + width: 50%; + min-width: 240px; + z-index: 1; + } + + /* tooltip content */ + #tooltip_content { + margin: 10px 0; + padding: 20px; + border-radius: 5px; + border: solid 1px #bdbdbd; + box-shadow: 0 0 20px rgba(0, 0, 0, 0.05); + background: #ffffff; + word-break: all; + word-break: break-word; + } + + /* tooltip copy of paragraphs and figures */ + #tooltip_content > p, + #tooltip_content > figure { + margin: 0; + } + + /* tooltip copy of figure captions */ + #tooltip_content > figure > figcaption { + margin: 0; + margin-top: 20px; + padding: 0; + } + + /* navigation bar */ + #tooltip_nav_bar { + margin-top: 10px; + text-align: center; + } + + /* navigation bar previous/next buton */ + #tooltip_nav_bar > .icon_button { + position: relative; + top: 3px; + } + + /* navigation bar previous button */ + #tooltip_nav_bar > .icon_button:first-of-type { + margin-right: 5px; + } + + /* navigation bar next button */ + #tooltip_nav_bar > .icon_button:last-of-type { + margin-left: 5px; + } + } + + /* always hide tooltip on print */ + @media only print { + #tooltip { + display: none; + } + } + + /* -------------------------------------------------- */ + /* jump to first plugin */ + /* -------------------------------------------------- */ + + @media only screen { + /* jump button */ + .jump_arrow { + position: relative; + top: 0.125em; + margin-right: 5px; + } + } + + /* always hide jump button on print */ + @media only print { + .jump_arrow { + display: none; + } + } + + /* -------------------------------------------------- */ + /* link highlight plugin */ + /* -------------------------------------------------- */ + + @media only screen { + /* anything with data-highlighted attribute true */ + [data-highlighted="true"] { + background: #ffeb3b; + } + + /* anything with data-selected attribute true */ + [data-selected="true"] { + background: #ff8a65 !important; + } + + /* animation definition for glow */ + @keyframes highlight_glow { + 0% { + background: none; + } + 10% { + background: #bbdefb; + } + 100% { + background: none; + } + } + + /* anything with data-glow attribute true */ + [data-glow="true"] { + animation: highlight_glow 2s; + } + } + + /* -------------------------------------------------- */ + /* table of contents plugin */ + /* -------------------------------------------------- */ + + @media only screen { + /* toc panel when open */ + #toc_panel { + box-sizing: border-box; + position: fixed; + top: 0; + left: 0; + min-width: 260px; + max-width: 480px; + /* keep panel edge consistent distance away from "page" edge */ + width: calc(((100vw - 8.5in) / 2) - 30px - 40px); + bottom: 0; + background: #ffffff; + border-right: solid 1px #bdbdbd; + box-shadow: 0 0 20px rgba(0, 0, 0, 0.05); + z-index: 2; + } + + /* toc panel when closed */ + #toc_panel[data-open="false"] { + min-width: 60px; + width: 60px; + height: 60px; + border-bottom: solid 1px #bdbdbd; + } + + /* toc panel header */ + #toc_panel > h3 { + box-sizing: border-box; + height: 60px; + margin: 0; + padding: 20px; + border-bottom: solid 1px #bdbdbd; + } + + /* toc panel when hovered */ + #toc_panel > h3:hover { + cursor: pointer; + } + + /* toc open/close header button */ + #toc_button { + margin-right: 20px; + vertical-align: top; + } + + /* toc header text */ + #toc_header_text { + vertical-align: top; + font-weight: 600; + position: relative; + top: -4px; + } + + /* hide toc list and header text when closed */ + #toc_panel[data-open="false"] #toc_header_text, + #toc_panel[data-open="false"] > #toc_list { + display: none; + } + + /* toc list of entries */ + #toc_list { + box-sizing: border-box; + width: 100%; + padding: 20px; + position: absolute; + top: calc(60px + 1px); + bottom: 0; + overflow: auto; + } + + /* toc entry, link to section in document */ + .toc_link { + display: block; + padding: 5px; + position: relative; + font-weight: 600; + text-decoration: none; + } + + /* toc entry when hovered or when "viewed" */ + .toc_link:hover, + .toc_link[data-viewing="true"] { + background: #f5f5f5; + } + + /* toc entry, level 1 indentation */ + .toc_link[data-level="1"] { + margin-left: 0; + } + + /* toc entry, level 2 indentation */ + .toc_link[data-level="2"] { + margin-left: 20px; + } + + /* toc entry, level 3 indentation */ + .toc_link[data-level="3"] { + margin-left: 40px; + } + + /* toc entry, level 4 indentation */ + .toc_link[data-level="4"] { + margin-left: 60px; + } + + /* toc entry bullets */ + #toc_panel[data-bullets="true"] .toc_link[data-level]:before { + position: absolute; + left: -15px; + top: -1px; + font-size: 1.5em; + } + + /* toc entry, level 2 bullet */ + #toc_panel[data-bullets="true"] .toc_link[data-level="2"]:before { + content: "\2022"; + } + + /* toc entry, level 3 bullet */ + #toc_panel[data-bullets="true"] .toc_link[data-level="3"]:before { + content: "\25AB"; + } + + /* toc entry, level 4 bullet */ + #toc_panel[data-bullets="true"] .toc_link[data-level="4"]:before { + content: "-"; + } + } + + /* when on screen < 8.5in wide */ + @media only screen and (max-width: 8.5in) { + /* push ("page") element down to make room for toc icon */ + .toc_body_nudge { + padding-top: 60px; + } + + /* toc icon when panel closed and not hovered */ + #toc_panel[data-open="false"]:not(:hover) { + background: rgba(255, 255, 255, 0.75); + } + } + + /* always hide toc panel on print */ + @media only print { + #toc_panel { + display: none; + } + } + + /* -------------------------------------------------- */ + /* lightbox plugin */ + /* -------------------------------------------------- */ + + @media only screen { + /* regular in document when hovered */ + .lightbox_img:hover { + cursor: pointer; + } + + .body_no_scroll { + overflow: hidden !important; + } + + /* screen overlay */ + #lightbox_overlay { + display: flex; + flex-direction: column; + position: fixed; + left: 0; + top: 0; + right: 0; + bottom: 0; + background: rgba(0, 0, 0, 0.75); + z-index: 3; + } + + /* middle area containing lightbox image */ + #lightbox_image_container { + flex-grow: 1; + display: flex; + justify-content: center; + align-items: center; + overflow: hidden; + position: relative; + padding: 20px; + } + + /* bottom area containing caption */ + #lightbox_bottom_container { + display: flex; + justify-content: center; + align-items: center; + height: 100px; + min-height: 100px; + max-height: 100px; + background: rgba(0, 0, 0, 0.5); + } + + /* image number info text box */ + #lightbox_number_info { + position: absolute; + color: #ffffff; + font-weight: 600; + left: 2px; + top: 0; + z-index: 4; + } + + /* zoom info text box */ + #lightbox_zoom_info { + position: absolute; + color: #ffffff; + font-weight: 600; + right: 2px; + top: 0; + z-index: 4; + } + + /* copy of image caption */ + #lightbox_caption { + box-sizing: border-box; + display: inline-block; + width: 100%; + max-height: 100%; + padding: 10px 0; + text-align: center; + overflow-y: auto; + color: #ffffff; + } + + /* navigation previous/next button */ + .lightbox_button { + width: 100px; + height: 100%; + min-width: 100px; + min-height: 100%; + color: #ffffff; + } + + /* navigation previous/next button when hovered */ + .lightbox_button:hover { + background: none !important; + } + + /* navigation button icon */ + .lightbox_button > svg { + height: 25px; + } + + /* figure auto-number */ + #lightbox_caption > span:first-of-type { + font-weight: bold; + margin-right: 5px; + } + + /* lightbox image when hovered */ + #lightbox_img:hover { + cursor: grab; + } + + /* lightbox image when grabbed */ + #lightbox_img:active { + cursor: grabbing; + } + } + + /* when on screen < 480px wide */ + @media only screen and (max-width: 480px) { + /* make navigation buttons skinnier on small screens to make more room for caption text */ + .lightbox_button { + width: 50px; + min-width: 50px; + } + } + + /* always hide lightbox on print */ + @media only print { + #lightbox_overlay { + display: none; + } + } + + /* -------------------------------------------------- */ + /* hypothesis (annotations) plugin */ + /* -------------------------------------------------- */ + + /* side panel */ + .annotator-frame { + width: 280px !important; + z-index: 0 !important; + } + + /* match highlight color to rest of theme */ + .annotator-highlights-always-on .annotator-hl { + background-color: #ffeb3b !important; + } + + /* match focused color to rest of theme */ + .annotator-hl.annotator-hl-focused { + background-color: #ff8a65 !important; + } + + /* match bucket bar color to rest of theme */ + .annotator-bucket-bar { + background: #f5f5f5 !important; + } + + /* always hide toolbar and tooltip on print */ + @media only print { + .annotator-frame { + display: none !important; + } + + hypothesis-adder { + display: none !important; + } + } From 3b5bf5c3c937a61ce08d7c8bfa8bf08a4b1af231 Mon Sep 17 00:00:00 2001 From: Vincent Rubinetti Date: Wed, 13 Mar 2019 17:41:01 -0400 Subject: [PATCH 215/216] Update content/02.delete-me.md to show formatting Merges https://github.com/manubot/rootstock/pull/169 --- content/02.delete-me.md | 264 ++++++++++++++++++++++++++++++++++++-- content/citation-tags.tsv | 2 +- content/metadata.yaml | 2 +- 3 files changed, 256 insertions(+), 12 deletions(-) diff --git a/content/02.delete-me.md b/content/02.delete-me.md index 9f0e403..bf51fa9 100644 --- a/content/02.delete-me.md +++ b/content/02.delete-me.md @@ -1,15 +1,259 @@ -## Manubot Rootstock Information +This manuscript is a template (aka "rootstock") for [Manubot](https://manubot.org/ "Manubot"), a tool for writing scholarly manuscripts. +Use this template as a starting point for your manuscript. -**Note: Manubot instances should delete this file.** +The rest of this document is a full list of formatting elements/features supported by Manubot. +Compare the input (`.md` files in the `/content` directory) to the output you see below. -The Manubot is a system for automating scholarly publishing. -Content is written in [Pandoc Markdown](http://pandoc.org/MANUAL.html#pandocs-markdown) source files. -See [`USAGE.md`](https://github.com/greenelab/manubot-rootstock/blob/master/USAGE.md) for more information on how to use the Manubot. +## Basic formatting -The Manubot project began with the [Deep Review](https://github.com/greenelab/deep-review), where it was used to compose a highly-collaborative review article [@doi:10.1098/rsif.2017.0387]. -Another example manuscript that was created with Manubot is: +**Bold** __text__ -+ The Sci-Hub Coverage Study ([GitHub](https://github.com/greenelab/scihub-manuscript), [HTML manuscript](https://greenelab.github.io/scihub-manuscript/)) [@doi:10.7554/eLife.32822] +[Semi-bold text]{.semibold} -If you notice a problem with Manubot, it's best to submit an upstream fix to the appropriate repository: -[`greenelab/manubot-rootstock`](https://github.com/greenelab/manubot-rootstock) for the git repository stub or [`greenelab/manubot`](https://github.com/greenelab/manubot) for the Python package. +[Centered text]{.center} + +[Right-aligned text]{.right} + +*Italic* _text_ + +Combined *italics and __bold__* + +~~Strikethrough~~ + +1. Ordered list item +2. Ordered list item + a. Sub-item + b. Sub-item + i. Sub-sub-item +3. Ordered list item + a. Sub-item + +- List item +- List item +- List item + +superscript2 + +subscript2n+1 + +[unicode superscripts](https://www.google.com/search?q=superscript+generator)⁰¹²³⁴⁵⁶⁷⁸⁹ + +[unicode subscripts](https://www.google.com/search?q=superscript+generator)₀₁₂₃₄₅₆₇₈₉ + +A long paragraph of text. +Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. +Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. +Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. +Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. + + +Putting each sentence on its own line has numerous benefits with regard to [editing](https://asciidoctor.org/docs/asciidoc-recommended-practices/#one-sentence-per-line) and [version control](https://rhodesmill.org/brandon/2012/one-sentence-per-line/). + +Line break without starting a new paragraph by putting +two spaces at end of line. + +## Document organization + +Document section headings: + +# Heading 1 + +## Heading 2 + +### Heading 3 + +#### Heading 4 + +### A heading centered on its own printed page{.center .page_center} + + + +Horizontal rule: + +--- + +`Heading 1`'s are recommended to be reserved for the title of the manuscript. + +`Heading 2`'s are recommended for broad sections such as *Abstract*, *Methods*, *Conclusion*, etc. + +`Heading 3`'s and `Heading 4`'s are recommended for sub-sections. + +## Links + +Bare URL link: + +[Link with text](https://manubot.org) + +[Link with hover text](https://manubot.org "Manubot Homepage") + +[Link by reference][manubot homepage] + +[Manubot Homepage]: https://manubot.org + +## Citations + +Citation by DOI [@doi:10.7554/eLife.32822]. + +Citation by PubMed Central ID [@pmcid:PMC6103790]. + +Citation by PubMed ID [@pmid:30718888]. + +Citation by Wikidata ID [@wikidata:Q56458321]. + +Citation by ISBN [@isbn:9780262517638]. + +Citation by URL [@url:https://github.com/manubot/rootstock]. + +Citation by tag [@tag:deep-review]. + +Multiple citations can be put inside the same set of brackets [@doi:10.7554/eLife.32822; @tag:deep-review; @isbn:9780262517638]. +Manubot plugins provide easier, more convenient visualization of and navigation between citations [@url:https://greenelab.github.io/meta-review; @pmid:30718888; @pmcid:PMC6103790; @tag:deep-review]. + +## Referencing figures, tables, equations + +Figure @fig:square-image + +Figure @fig:wide-image + +Figure @fig:tall-image + +Figure @fig:vector-image + +Table @tbl:bowling-scores + +Equation @eq:regular-equation + +Equation @eq:long-equation + +## Quotes and code + +> Quoted text + + +> Quoted block of text +> +> Two roads diverged in a wood, and I— +> I took the one less traveled by, +> And that has made all the difference. + + +Code `in the middle` of normal text, aka `inline code`. + +Code block with Python syntax highlighting: + +```python +from manubot.cite.doi import expand_short_doi + +def test_expand_short_doi(): + doi = expand_short_doi("10/c3bp") + # a string too long to fit within page: + assert doi == "10.25313/2524-2695-2018-3-vliyanie-enhansera-copia-i-insulyatora-gypsy-na-sintez-ernk-modifikatsii-hromatina-i-svyazyvanie-insulyatornyh-belkov-vtransfetsirovannyh-geneticheskih-konstruktsiyah" +``` + +Code block with no syntax highlighting: + +``` +Exporting HTML manuscript +Exporting DOCX manuscript +Exporting PDF manuscript +``` + +## Figures + +![ +**A square image at actual size and with a bottom caption.** +Loaded from the latest version of image on GitHub. +](https://raw.githubusercontent.com/greenelab/manubot-resources/master/test/square.png "Square image"){#fig:square-image} + +![ +**An image too wide to fit within page at full size.** +Loaded from a specific (hashed) version of the image on GitHub. +](https://raw.githubusercontent.com/manubot/resources/bfd6afcd9f47d7da34362e62a005d86e85aef25a/test/wide.png "Wide image"){#fig:wide-image} + +![ +**A tall image with a specified height.** +Loaded from a specific (hashed) version of the image on GitHub. +](https://raw.githubusercontent.com/manubot/resources/bfd6afcd9f47d7da34362e62a005d86e85aef25a/test/tall.png "Tall image"){#fig:tall-image height=3in} + +![ +**A vector `.svg` image loaded from GitHub.** +The parameter `sanitize=true` is necessary to properly load SVGs hosted via GitHub URLs. +White background specified to serve as a backdrop for transparent sections of the image. +](https://raw.githubusercontent.com/manubot/resources/master/test/vector.svg?sanitize=true "Vector image"){#fig:vector-image height=2.5in .white} + +## Tables + +| *Bowling Scores* | Jane | John | Alice | Bob | +|:-----------------|:-------------:|:-------------:|:-------------:|:-------------:| +| Game 1 | 150 | 187 | 210 | 105 | +| Game 2 | 98 | 202 | 197 | 102 | +| Game 3 | 123 | 180 | 238 | 134 | + +Table: A table with a top caption and specified relative column widths. +{#tbl:bowling-scores} + +| | Digits 1-33 | Digits 34-66 | Digits 67-99 | Ref. | +|:--------|:-----------------------------------|:----------------------------------|:----------------------------------|:----------------------------| +| pi | 3.14159265358979323846264338327950 | 288419716939937510582097494459230 | 781640628620899862803482534211706 | [`piday.org`](https://www.piday.org/million/) | +| e | 2.71828182845904523536028747135266 | 249775724709369995957496696762772 | 407663035354759457138217852516642 | [`nasa.gov`](https://apod.nasa.gov/htmltest/gifcity/e.2mil) | + +Table: A table too wide to fit within page. +{#tbl:constant-digits} + +## Equations + +A LaTeX equation: + +$$\int_0^\infty e^{-x^2} dx=\frac{\sqrt{\pi}}{2}$$ {#eq:regular-equation} + +An equation too long to fit within page: + +$$x = a + b + c + d + e + f + g + h + i + j + k + l + m + n + o + p + q + r + s + t + u + v + w + x + y + z + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9$$ {#eq:long-equation} + +## Special + + [WARNING]{.semibold} _The following features are only supported and intended for `.html` and `.pdf` exports._ +_Journals are not likely to support them, and they may not display correctly when converted to other formats such as `.docx`._ + +[Link styled as a button](https://manubot.org "Manubot Homepage"){.button} + +Available background colors for text, images, code, banners, etc: + +`white`{.white} +`lightgrey`{.lightgrey} +`grey`{.grey} +`darkgrey`{.darkgrey} +`black`{.black} +`lightred`{.lightred} +`lightyellow`{.lightyellow} +`lightgreen`{.lightgreen} +`lightblue`{.lightblue} +`lightpurple`{.lightpurple} +`red`{.red} +`orange`{.orange} +`yellow`{.yellow} +`green`{.green} +`blue`{.blue} +`purple`{.purple} + +Using the [Font Awesome](https://fontawesome.com/) icon set: + + + + + + +[ + **Light Grey Banner**
+useful for *general information* - [manubot.org](https://manubot.org/) +]{.banner .lightgrey} + +[ + **Blue Banner**
+useful for *important information* - [manubot.org](https://manubot.org/) +]{.banner .lightblue} + +[ + **Light Red Banner**
+useful for *warnings* - [manubot.org](https://manubot.org/) +]{.banner .lightred} diff --git a/content/citation-tags.tsv b/content/citation-tags.tsv index 6de0742..94fd8ee 100644 --- a/content/citation-tags.tsv +++ b/content/citation-tags.tsv @@ -1,2 +1,2 @@ tag citation -deep_review doi:10.1101/142760 +deep-review doi:10.1098/rsif.2017.0387 diff --git a/content/metadata.yaml b/content/metadata.yaml index 31e6adb..553de29 100644 --- a/content/metadata.yaml +++ b/content/metadata.yaml @@ -1,5 +1,5 @@ --- -title: "Manubot Rootstock: Manuscript Title" +title: "Manuscript Title" keywords: - markdown - publishing From b422130eb46ea5d188779910772192730d79b172 Mon Sep 17 00:00:00 2001 From: Vincent Rubinetti Date: Wed, 13 Mar 2019 17:51:46 -0400 Subject: [PATCH 216/216] CSS: background colors, alignment classes, print control Merges https://github.com/manubot/rootstock/pull/188 Miscellaneous changes to default.html style including background colors, text/page alignment classes, print control styles. Switch margins to 0.75 inches to reduce pages needed to print manuscript. --- build/themes/default.html | 109 ++++++++++++++++++++++++++++---------- 1 file changed, 81 insertions(+), 28 deletions(-) diff --git a/build/themes/default.html b/build/themes/default.html index b65d6fe..a4adf6b 100644 --- a/build/themes/default.html +++ b/build/themes/default.html @@ -27,7 +27,7 @@ position: relative; box-sizing: border-box; max-width: 8.5in; - font-size: 16px; + font-size: 12pt; line-height: 1.5; margin: 20px auto; padding: 40px; @@ -53,20 +53,6 @@ } } - @media only print { - /* "page" element */ - body { - margin: 0; - padding: 0; - font-size: 12px; - } - } - - @page { - /* suggested printing margin */ - margin: 0.5in; - } - /* -------------------------------------------------- */ /* headings */ /* -------------------------------------------------- */ @@ -139,6 +125,33 @@ font-weight: 600; } + /* class for styling elements horizontally left aligned */ + .left { + display: block; + text-align: left; + margin-left: auto; + margin-right: 0; + justify-content: left; + } + + /* class for styling elements horizontally centered */ + .center { + display: block; + text-align: center; + margin-left: auto; + margin-right: auto; + justify-content: center; + } + + /* class for styling elements horizontally right aligned */ + .right { + display: block; + text-align: right; + margin-left: 0; + margin-right: auto; + justify-content: right; + } + /* -------------------------------------------------- */ /* section elements */ /* -------------------------------------------------- */ @@ -310,9 +323,21 @@ /* highlight colors */ /* -------------------------------------------------- */ + .white { + background: #ffffff; + } .lightgrey { background: #eeeeee; } + .grey { + background: #757575; + } + .darkgrey { + background: #424242; + } + .black { + background: #000000; + } .lightred { background: #ffcdd2; } @@ -328,9 +353,6 @@ .lightpurple { background: #f3e5f5; } - .grey { - background: #757575; - } .red { background: #f44336; } @@ -349,6 +371,7 @@ .purple { background: #9c27b0; } + .white, .lightgrey, .lightred, .lightyellow, @@ -357,6 +380,7 @@ .lightpurple, .orange, .yellow, + .white a, .lightgrey a, .lightred a, .lightyellow a, @@ -368,10 +392,15 @@ color: #000000; } .grey, + .darkgrey, + .black, .red, .green, .blue, .purple, + .grey a, + .darkgrey a, + .black a, .red a, .green a, .blue a, @@ -449,39 +478,63 @@ } /* -------------------------------------------------- */ - /* page break control */ + /* print control */ /* -------------------------------------------------- */ @media print { - /* all

headings except the first */ - h2:not(:first-of-type) { - /* force page break */ - break-before: always !important; + @page { + /* suggested printing margin */ + margin: 0.75in; + } + + /* document and "page" elements */ + html, body { + margin: 0; + padding: 0; + width: 100%; + height: 100%; + } + + /* class for centering an element vertically on its own page */ + .page_center { + margin: auto; + width: 100%; + height: 100%; + display: flex; + align-items: center; + vertical-align: middle; + break-before: page; + break-after: page; + } + + /*

heading */ + h2 { + margin-top: 0; } /* always insert a page break before the element */ .page_break_before { - break-before: always !important; + break-before: page; } /* always insert a page break after the element */ .page_break_after { - break-after: always !important; + break-after: page; } /* avoid page break before the element */ .page_break_before_avoid { - break-before: avoid !important; + break-before: avoid; } /* avoid page break after the element */ .page_break_after_avoid { - break-after: avoid !important; + break-after: avoid; } /* avoid page break inside the element */ .page_break_inside_avoid { - break-inside: avoid !important; + break-inside: avoid; } }