From f2e69099c928de05357e1b3e55e867f8f189a879 Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Fri, 1 Nov 2024 13:02:32 -0700 Subject: [PATCH] Release 1.1.0 :tada: (#57) * feat: support Python 3.7+; drop Python 2.7,<3.7 * feat: pass through json serialization options * docs: add advanced usage examples * feat: spec-compliant routes, with legacy support * fix: remove debug messages * test: increase unit test coverage (#52) * test: increase unit test coverage * Update tests.yml --------- Co-authored-by: Liam Beckman * docs: browsable API reference on GitHub Pages (#49) * docs: browsable API reference on GitHub Pages * Update tests.yml --------- Co-authored-by: Liam Beckman * Minor release testing updates * Update support for Service Info (#55) - Fix version to match PyPi release (#59) * Add integration test with Funnel * Add initial TES integration test (Funnel) * Add Tmate debug session * Update Funnel installation * Re-add all unit tests * Fix Funnel download issue with correct rc version * Add latest stable Python to test matrix * Fix unit tests not reaching Funnel * Increase test coverage * Minor linting fix * Update raised exception for invalid server response * Update README * Update README.md * Exclude tests from packages (#58) * Add in 3 fields missing in the models for tes. --------- Co-authored-by: Alex Kanitz Co-authored-by: Kyle Ellrott Co-authored-by: Ben Beasley Co-authored-by: Venkat Malladi --- .coverage | 1 - .github/workflows/docs.yml | 36 + .github/workflows/tests.yml | 59 +- .gitignore | 178 ++++- .travis.yml | 22 - README.md | 154 ++++- docs/.pages | 3 + docs/README.md | 149 +++++ docs/docstring/.pages | 4 + docs/docstring/README.md | 48 ++ docs/docstring/client.md | 186 ++++++ docs/docstring/models.md | 1052 ++++++++++++++++++++++++++++++ docs/docstring/utils.md | 117 ++++ docs/requirements.txt | 5 + mkdocs.yml | 15 + requirements.txt | 7 +- setup.py | 23 +- tes/__init__.py | 36 +- tes/client.py | 270 ++++++-- tes/models.py | 360 +++++++--- tes/utils.py | 62 +- tests/__init__.py | 0 tests/integration/test_funnel.py | 35 + tests/requirements.txt | 11 +- tests/test_client.py | 225 ++++++- tests/test_models.py | 265 ++++++-- tests/test_utils.py | 135 +++- 27 files changed, 3096 insertions(+), 362 deletions(-) delete mode 100644 .coverage create mode 100644 .github/workflows/docs.yml delete mode 100644 .travis.yml create mode 100644 docs/.pages create mode 100644 docs/README.md create mode 100644 docs/docstring/.pages create mode 100644 docs/docstring/README.md create mode 100644 docs/docstring/client.md create mode 100644 docs/docstring/models.md create mode 100644 docs/docstring/utils.md create mode 100644 docs/requirements.txt create mode 100644 mkdocs.yml create mode 100644 tests/__init__.py create mode 100644 tests/integration/test_funnel.py diff --git a/.coverage b/.coverage deleted file mode 100644 index 39575c3..0000000 --- a/.coverage +++ /dev/null @@ -1 +0,0 @@ -!coverage.py: This is a private format, don't read it directly!{"lines":{"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/requests/utils.py":[9,11,12,13,14,15,16,17,18,19,20,21,534,23,24,26,27,28,32,33,34,37,39,42,561,521,572,583,524,695,90,607,784,99,100,101,103,104,615,620,622,157,616,629,532,635,533,640,641,130,619,644,154,538,160,674,163,164,166,168,169,170,683,686,642,177,692,182,183,698,546,211,724,730,219,733,737,738,739,740,741,745,244,680,639,259,262,265,266,779,780,781,272,816,304,833,839,841,842,843,844,846,850,851,854,861,863,864,867,868,875,364,268,379,892,390,410,417,419,420,431,449,459,675,676,496,497,339,500,506,507],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/requests/sessions.py":[517,9,10,11,12,13,14,16,17,18,20,21,22,23,24,27,28,30,35,38,551,41,47,562,56,57,59,60,573,64,65,69,70,583,74,75,78,591,592,81,597,87,88,601,602,603,606,96,528,98,612,615,616,619,622,111,113,114,628,117,631,120,634,637,645,651,652,654,656,663,665,666,667,672,673,674,539,677,678,679,680,682,683,685,699,701,702,704,709,710,712,715,719,724,221,245,286,50,309,326,329,330,331,334,339,343,348,351,356,359,362,366,372,609,376,382,385,386,387,389,390,392,393,395,405,408,409,412,413,416,417,418,420,421,422,423,424,425,426,427,428,429,430,431,433,436,437,438,100,590,476,477,478,479,480,481,482,483,484,485,486,593,488,490,492,493,497,498,499,501,502,504,506,119],"/Users/strucka/Projects/tes-python-packages/py-tes/tes/__init__.py":[1,3,4,8,9,10,11,12,13,14,15,16,17,18],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/urllib3/fields.py":[1,2,3,5,71,8,105,138,50,116,158,22,157,62,63],"/Users/strucka/Projects/tes-python-packages/py-tes/tests/test_client.py":[1,2,3,4,7,8,11,12,14,15,16,20,21,22,24,25,26,27,28,29,31,32,33,35,36,37,39,40,42,43,44,45,46,47,48,49,52,53,54,55,57,59,60,61,63,64,66,67,68,69,70,71,72,75,76,77,78,80,82,83,84,86,87,89,90,91,92,93,94,96,97,98,99,101,103,104,105,107,108,110,111,112,113,114,115,117,118,119,120,122,124,125,126,128,129],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/requests/adapters.py":[9,11,12,14,15,16,17,18,19,20,21,22,23,24,25,26,27,29,30,31,34,35,36,38,40,41,42,43,46,47,48,49,52,53,55,56,58,59,76,81,106,107,108,110,111,112,113,114,117,118,120,122,123,124,126,128,132,144,157,158,159,161,162,164,201,253,263,266,269,272,273,274,276,277,282,285,286,288,290,313,319,320,323,352,366,388],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/chardet/sbcsgroupprober.py":[34,35,37,38,39,40,43,44,29,30,31],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/yaml/events.py":[4,5,6,7,8,15,16,21,22,23,24,25,26,27,28,29,31,32,36,37,38,39,40,42,43,45,46,47,48,49,50,51,52,54,55,56,57,58,59,61,62,64,66,67,68,69,70,71,72,73,75,76,78,79,81,82,84,85],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/nose/plugins/skip.py":[57,59,60,61],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/nose/proxy.py":[30,163,164,165,168,169,170,43,45,46,47,176,177,178,57,58,59,60,62,64,78,80,81,82,83,102,103,104,110,111,112,116,117,118],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/chardet/langturkishmodel.py":[192,52,183,186,187,188,189,190,191],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/chardet/version.py":[8,9,6],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/urllib3/__init__.py":[3,5,6,8,14,15,16,17,18,19,20,21,25,26,27,33,34,35,51,54,57,76,83,85,87,88,90,93],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/six.py":[1,23,25,26,27,28,29,31,32,36,37,38,40,49,50,51,52,53,55,60,62,63,64,65,71,72,75,77,80,82,83,86,88,89,91,92,93,94,97,100,103,105,106,107,112,114,115,117,124,126,127,128,130,136,139,141,142,143,154,155,156,157,159,160,161,164,171,173,174,175,177,178,179,181,182,184,185,186,187,189,190,191,195,196,198,199,200,201,202,205,206,207,209,218,224,226,229,231,232,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,288,289,290,291,292,293,294,295,296,297,298,299,300,303,308,309,310,311,312,314,316,317,320,322,326,327,328,329,330,331,332,333,334,335,336,337,338,339,340,341,342,343,344,345,346,347,348,350,351,352,354,356,357,360,362,366,367,368,370,371,372,374,376,377,380,382,386,387,388,389,390,391,392,393,394,395,396,397,398,399,400,401,402,403,404,405,406,407,408,409,410,411,412,413,414,415,416,417,418,420,421,422,424,426,427,430,432,436,437,438,439,441,442,443,445,447,448,451,453,457,459,460,461,463,465,466,469,471,472,473,474,475,476,477,479,482,483,486,491,502,511,512,514,515,516,517,520,521,525,528,529,535,546,549,552,555,557,560,561,562,565,566,567,568,569,570,573,592,595,598,599,601,604,606,608,610,611,612,613,614,615,618,642,643,646,648,649,651,654,656,657,658,659,660,661,662,663,666,670,674,678,689,691,692,693,694,695,696,699,701,703,706,712,717,721,722,776,777,779,786,788,789,790,800,812,828,849,850,851,856,857,862,863,866,868],"/Users/strucka/Projects/tes-python-packages/py-tes/tes/models.py":[1,3,5,6,9,10,11,13,17,18,19,21,22,23,24,25,28,35,36,39,40,41,42,43,44,45,48,51,52,54,55,56,57,60,61,62,66,67,68,69,71,72,74,75,77,78,80,81,83,84,88,89,90,91,93,94,96,97,99,100,102,103,107,108,109,110,113,114,115,116,118,119,121,122,124,125,127,128,130,131,133,134,136,137,141,142,143,144,146,147,149,150,152,153,155,156,158,159,161,162,166,167,168,169,171,172,174,175,179,180,181,182,184,185,187,188,190,191,193,194,198,199,200,201,203,204,205,206,207,210,211,213,214,216,217,219,220,222,223,225,226,228,229,231,232,234,235,237,238,241,242,245,246,253,254,255,256,257,268,278,281,282,283,284,286,287,291,292,293,294,298,299,300,303,304,305,306,308,309,311,312,316,317,318,319,323,324,325,328,329,330,331,333,334,336,337,339,340,342,343,347,348,349,350,352,353],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/yaml/error.py":[2,4,37,6,7,8,9,10,11,12,45,14,48,50,51,46,58],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/nose/plugins/capture.py":[64,96,98,69,102,97,58,59,101],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/attr/validators.py":[128,130,3,5,134,7,137,138,11,12,13,14,15,144,146,19,20,21,23,27,133,36,166,39,43,151,56,59,60,61,63,147,75,82,135,98,99,100,38,102,103,104,106,108,110,111,115],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/nose/suite.py":[539,540,543,544,545,546,547,548,551,552,554,563,52,53,567,68,72,73,75,76,79,80,81,94,95,96,97,98,99,100,103,104,105,106,107,113,114,148,149,150,151,153,154,155,156,157,158,173,177,201,204,205,208,209,216,217,218,224,226,227,228,269,270,274,277,278,279,282,283,285,286,287,288,289,290,291,292,293,297,298,301,302,303,304,308,309,310,312,564,315,323,324,325,326,327,328,329,330,331,337,338,339,340,341,342,313,345,346,347,348,349,350,351,356,357,358,360,361,362,364,365,367,368,372,373,374,394,396,397,401,402,403,404,405,406,407,418,419,420,421,422,423,424,427,435,436,441,443,445,446,447,448,451,452,453,454,457,459,460,462,463,464,465,466,467,471,474,475,476,477,478,479,480,481,482,483,484,485,486],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/nose/plugins/testid.py":[137,138,142,143,144,145,148,149,150,151,154,155],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/urllib3/packages/ssl_match_hostname/_implementation.py":[96,1,6,7,14,15,16,17,19,84,22,23,26,79],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/chardet/utf8prober.py":[35,36,38,76,44,49,53,57,28,29,30,31],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/yaml/resolver.py":[2,4,5,7,9,10,23,12,14,15,16,164,18,19,21,22,217,25,26,27,28,154,30,31,160,33,34,35,162,37,167,168,170,171,172,174,175,177,178,179,219,183,184,116,186,187,188,192,193,195,196,197,198,161,200,201,202,143,205,207,208,209,163,213,214,216,89,218,91,92,93,223,224,225,226,165,144,114,115,204,120],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/chardet/gb2312prober.py":[33,34,40,44,28,29,30,31],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/idna/__init__.py":[1],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/requests/packages.py":[1,6,7,10,11,12],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/urllib3/util/timeout.py":[1,99,4,5,7,103,11,140,15,18,195,171,182,88,91,156,93,213],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/chardet/enums.py":[5,8,11,12,13,14,17,21,22,23,24,25,26,27,28,29,32,35,36,37,38,41,44,45,46,47,50,53,54,55,56,57,59,65,71,72,73,74,75,76],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/nose/plugins/xunit.py":[192,193,191],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/chardet/sjisprober.py":[32,33,36,37,44,48,52,56,89,28,29,30,31],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/requests/api.py":[129,11,101,71,72,112,75,13,143,16,115,88,57,58,61],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/chardet/mbcsgroupprober.py":[32,33,34,35,36,37,38,41,42,30,31],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/attr/_config.py":[1,19,4,6,9],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/requests/__init__.py":[41,43,44,45,46,49,50,51,54,58,59,61,62,63,66,67,69,70,71,75,76,83,84,86,87,90,91,93,94,95,97,98,99,100,101,102,103,110,111,112,118,121],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/urllib3/contrib/pyopenssl.py":[43,44,46],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/urllib3/poolmanager.py":[1,2,3,4,6,7,8,9,10,11,12,13,16,402,19,149,22,151,152,281,410,155,154,159,160,162,165,425,170,301,49,54,439,57,266,147,197,204,206,377,352,229,110,111,112,242,115,116,117,153,121,379,380],"/Users/strucka/Projects/tes-python-packages/py-tes/tests/test_utils.py":[1,2,4,5,8,10,11,12,13,14,15,16,18,19,20,21,22,24,25,26,27,28,29,30,31,33,34,36,37,38,39,43,44,45,49,50,51,52,53,54,55,56],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/pkg_resources/_vendor/six.py":[185,187],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/chardet/hebrewprober.py":[128,130,131,132,133,134,135,136,137,138,139,144,149,151,152,154,28,29,286,164,174,178,182,196,282,255],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/requests_mock/exceptions.py":[14,15,18,19,21,24,29,30],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/yaml/representer.py":[2,3,5,6,8,10,12,13,15,17,18,20,27,34,40,74,75,76,77,78,80,81,82,83,84,86,94,112,136,139,141,149,153,169,172,179,182,185,186,187,189,209,224,227,233,237,241,248,251,252,254,255,257,258,260,261,263,264,266,267,269,270,272,273,275,276,278,279,281,282,284,285,287,288,290,291,293,295,311,320,326,337,340,344,348,389,451,452,454,455,457,458,460,461,463,464,466,467,469,470,472,473,475,476,478,479,481,482,484,485],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/requests_mock/compat.py":[32,34,60,42,44,13,47,16,17,20,23,25,26,31,28,29,30,52],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/idna/core.py":[1,2,3,4,5,6,8,9,10,12,258,16,17,18,131,21,22,23,26,27,28,286,31,32,33,36,39,42,45,49,307,56,190,63,140,335,231,364,146,124],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/yaml/constructor.py":[2,3,516,5,6,8,10,12,13,15,17,18,20,21,22,535,24,26,30,35,37,38,551,42,43,44,45,46,47,48,50,51,52,565,55,56,58,61,574,64,65,66,67,68,633,524,106,98,87,88,100,91,92,93,94,101,609,668,99,612,613,614,103,616,105,618,620,621,622,112,625,626,628,629,630,617,120,532,634,636,637,638,127,640,641,642,644,645,646,135,648,137,650,23,652,653,654,656,657,658,660,149,150,151,152,153,666,155,156,157,158,159,672,161,674,163,164,113,168,170,171,172,173,174,175,117,197,201,202,205,206,207,208,632,210,214,215,216,217,218,219,220,223,227,39,110,125,126,255,256,257,258,260,129,128,121,670,284,133,292,134,302,304,649,624,53,334,356,377,383,384,385,386,662,390,391,392,393,395,396,397,398,399,661,401,665,411,416,417,418,420,421,422,424,425,426,428,429,430,669,432,433,434,436,437,438,440,441,442,444,445,446,448,449,450,452,453,454,673,456,457,458,460,461,462,464,465,467,469,472,475,118,478,481,484,664,495],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/requests_mock/response.py":[13,15,16,17,18,19,20,22,23,25,26,28,29,32,34,60,62,64,68,70,75,76,78,83,87,96,97,98,102,106,112,114,115,119,122,139,141,143,144,145,146,147,148,150,152,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,171,172,173,175,177,180,181,183,184,185,186,187,190,192,193,198,201,202,207,208,210,214,218,220,225,226,229,230,231,232,235,236,238,239,240,241,242,243,244,245,246,247],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/idna/intranges.py":[34,6,38,8,10,31],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/nose/plugins/failuredetail.py":[33,35,36],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/nose/plugins/logcapture.py":[34,38,39,40,41,44,178,179,193,194,195,196,198,199,76,204,77,78,207,80,209,86,217,79,222,208],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/urllib3/packages/__init__.py":[1,3,5],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/chardet/compat.py":[22,25,26,27,28,29],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/nose/pyversion.py":[70,136,49,50,51,52,53,54,56,58],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/urllib3/packages/ssl_match_hostname/__init__.py":[1,3,6,7,10,11,13,14,16,19],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/urllib3/util/request.py":[1,2,4,5,7,8,11,12,77,95],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/yaml/__init__.py":[128,129,2,4,5,6,8,9,11,13,14,15,19,276,279,280,281,282,156,30,69,295,290,163,164,165,166,167,168,41,298,299,301,93,306,52,286,308,314,296,64,267,197,70,71,73,258,75,204,212,87,292,220,221,95,293,103,104,105,231,241,249,125,126,127],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/requests/models.py":[513,515,516,517,8,521,10,11,12,526,17,530,19,20,21,22,849,25,26,27,29,30,31,34,35,548,39,42,43,559,48,49,50,51,52,565,55,56,57,60,61,574,575,578,581,609,584,585,588,589,591,592,593,82,596,601,91,92,93,606,95,96,97,98,99,612,101,102,746,104,105,103,620,109,623,531,617,532,635,637,640,643,534,654,877,662,665,675,170,171,685,174,689,179,180,182,704,194,709,711,177,631,716,721,726,216,219,220,223,224,225,226,227,229,230,743,233,234,235,236,237,238,239,240,241,243,894,246,763,765,767,770,772,774,264,300,779,781,776,280,282,284,286,520,288,560,291,293,295,297,812,301,402,304,305,306,307,308,309,822,564,315,317,830,832,331,333,334,335,568,850,853,825,857,858,347,354,355,868,870,360,745,365,573,370,371,375,381,524,917,896,320,747,388,393,397,398,400,401,914,405,918,408,409,410,411,412,413,414,415,928,417,930,931,420,933,934,423,424,937,939,428,430,431,433,436,437,438,440,441,442,444,816,100,451,452,748,454,462,463,464,467,468,846,472,847,337,827,495,936,498,499,500,501,425,505,818,508,511],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/urllib3/request.py":[1,3,4,37,7,72,41,10,39,44,45,50,89,90,42],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/yaml/cyaml.py":[2,3,5,7,9,10,12,14,16,21,23,28,30,35,38,39,40,41,42,52,55,56,57,58,59,69,72,73,74,75,76],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/yaml/composer.py":[129,2,4,5,6,135,8,9,138,11,13,14,16,24,29,31,133,34,35,36,39,46,48,136,50,52,55,58,60,61,63,64,71,72,73,78,79,80,81,82,83,84,85,86,137,88,89,90,91,92,93,94,95,97,99,100,101,102,103,104,105,106,107,109,110,111,112,113,114,115,117,118,119,120,121,122,123,124,125,127],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/requests_mock/adapter.py":[13,15,16,17,19,20,21,23,26,28,29,31,32,34,37,38,42,46,50,55,62,65,66,68,76,78,79,80,81,82,83,84,87,88,89,90,91,92,94,95,96,104,105,108,109,113,114,118,122,126,129,133,134,136,143,148,150,151,171,173,174,177,180,182,183,184,185,186,188,189,194,197,200,202,203,206,209,210,211,212,213,215,216,217,218,219,221,222,223,228,229,230,231,235,238,244,245,246,247,249,252,255,256,264,265,266,267,268,269,270,271,272,273,274,276,284,287],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/chardet/charsetgroupprober.py":[32,33,65,39,49,85,57,28,29],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/chardet/charsetprober.py":[32,66,35,37,39,103,44,61,47,51,54,58,29,30],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/urllib3/contrib/socks.py":[32,33,37,39,23,24,26,27,28,29,30],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/nose/case.py":[128,129,130,131,132,133,140,147,148,149,151,29,33,34,36,37,38,39,40,41,42,45,59,60,64,69,70,74,99,100,101,102,103,104],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/requests/auth.py":[8,10,11,12,13,14,15,17,19,20,21,22,24,25,28,286,292,266,72,73,75,79,80,82,86,217,92,222,95,100,101,103,108,109,111,117,127],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/chardet/langhebrewmodel.py":[193,194,195,196,197,198,199,53,190],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/urllib3/packages/six.py":[1,2,4,6,8,10,12,13,20,23,24,25,26,27,28,29,31,32,35,36,37,38,39,40,41,45,47,49,50,51,52,53,55,57,59,60,61,62,63,64,65,67,69,71,72,73,75,77,79,80,81,82,83,85,86,87,88,89,91,92,93,94,97,100,103,105,106,107,112,114,115,117,124,126,127,128,130,136,139,141,142,143,154,155,156,157,159,160,161,164,171,173,174,175,177,178,179,181,182,184,185,186,187,189,190,191,195,196,198,199,200,201,202,203,205,206,207,209,218,224,226,229,231,232,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,288,289,290,291,292,293,294,295,296,297,298,299,300,303,308,309,310,311,312,314,316,317,320,322,326,327,328,329,330,331,332,333,334,335,336,337,338,339,340,341,342,343,344,345,346,347,348,350,351,352,354,356,357,360,362,366,367,368,370,371,372,374,376,377,380,382,386,387,388,389,390,391,392,393,394,395,396,397,398,399,400,401,402,403,404,405,406,407,408,409,410,411,412,413,414,415,416,417,418,420,421,422,424,426,427,430,432,436,437,438,439,441,442,443,445,447,448,451,453,457,459,460,461,463,465,466,469,471,472,473,474,475,476,477,479,482,483,486,491,502,511,512,514,515,516,517,520,521,525,528,529,535,546,549,552,555,557,560,561,562,565,566,567,568,569,570,573,592,595,596,598,601,604,606,608,610,611,612,613,614,615,618,642,646,648,649,651,654,656,657,658,659,660,661,662,663,666,670,674,678,689,691,692,693,694,695,696,699,701,703,706,712,717,721,722,776,777,779,786,788,789,790,800,812,828,849,850,851,856,857,862,863,866,868],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/attr/_make.py":[512,1,514,3,4,1029,6,519,8,9,10,523,524,526,941,344,18,19,20,21,22,345,536,25,540,541,30,31,545,34,37,40,553,554,43,44,557,46,349,560,50,563,513,565,566,1033,56,57,58,571,572,1034,693,352,579,326,694,1036,586,695,588,590,591,569,594,568,799,699,1041,617,716,530,570,531,717,532,635,637,126,533,130,131,132,645,134,135,136,137,138,142,365,803,707,789,662,755,152,154,155,157,158,159,160,161,164,165,166,167,170,199,177,178,179,180,542,182,183,696,185,186,543,671,714,373,193,195,708,197,516,712,201,202,715,204,546,718,207,208,209,210,211,724,377,217,218,719,220,638,549,720,587,379,230,564,237,238,239,754,723,756,757,758,761,765,766,575,790,642,558,333,794,559,796,797,798,133,800,801,802,219,804,805,806,812,813,814,815,818,825,223,828,906,832,181,834,323,836,837,838,839,840,841,842,843,844,845,334,847,336,337,850,1030,340,853,854,855,856,857,346,347,860,861,350,864,356,358,359,872,361,363,364,402,573,371,372,885,374,887,376,889,890,891,380,381,895,384,388,389,902,391,904,905,394,395,908,909,911,912,401,914,403,916,918,919,920,580,922,925,926,415,928,929,930,931,933,422,426,427,428,429,942,431,945,947,437,438,585,441,927,445,446,448,672,331,452,965,454,968,969,458,459,322,463,419,982,983,984,473,986,482,995,491,500,509,510,511],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/urllib3/packages/ordered_dict.py":[128,257,132,5,6,137,10,11,142,16,17,151,154,156,28,218,158,159,160,161,34,36,37,38,39,40,41,42,44,173,175,48,177,50,51,52,54,169,244,190,63,65,66,67,68,197,71,79,69,210,212,213,214,215,216,164,49,91,220,224,225,235,168,116,170,120,249,122,124,253,126],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/yaml/serializer.py":[2,4,5,6,8,9,74,11,13,46,15,16,78,36,27,60],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/urllib3/contrib/__init__.py":[1],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/chardet/euctwfreq.py":[385,44,47],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/nose/core.py":[65,34,36,37,41,42,43,44,50,51,55,56,59,188,61,62,193,66,199,200,201,202,203,204,205,207,187,60],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/chardet/mbcharsetprober.py":[34,37,39,45,53,57,90,61,30,31],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/attr/__init__.py":[1,3,10,20,24,25,26,27,30,32,33,34,35,37,38,40,41,44,45,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/chardet/euckrprober.py":[34,35,41,45,28,29,30,31],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/nose/loader.py":[522,523,540,541,542,547,550,551,559,564,566,567,569,570,79,81,82,83,84,85,86,89,90,91,92,93,94,95,97,99,105,108,109,110,112,113,114,116,119,121,122,123,128,131,134,135,143,144,145,146,147,149,150,151,154,156,157,158,159,160,161,163,164,167,168,170,177,178,179,180,181,182,183,186,196,197,200,201,209,210,211,212,314,315,316,317,321,322,323,325,326,327,328,330,331,332,333,338,340,341,343,356,359,369,371,374,375,378,379,404,405,406,409,410,416,417,418,420,421,428,431,432,433,434,435,436,447,454,455,473,474,475,476,481,486,487,488,493,494],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/chardet/langgreekmodel.py":[224,69,50,206,209,210,211,212,213,214,215,218,219,220,221,222,223],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/nose/plugins/cover.py":[164,263,173,271,182,183],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/certifi/__init__.py":[1,3],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/chardet/cp949prober.py":[34,35,43,47,28,29,30,31],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/urllib3/_collections.py":[1,2,3,4,133,262,135,136,265,138,139,268,14,15,18,276,21,150,151,24,281,154,27,157,261,160,237,294,39,168,41,43,172,173,47,175,48,177,50,180,182,137,57,87,288,224,72,202,247,45,79,208,83,142,86,185,89,90,143,92,93,229,96,263,101,232,171,44,146,238,297,250,234],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/attr/converters.py":[8,3,5],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/nose/selector.py":[149,129,130,131,134,135,140,141,144,148,171,153,152,68,154,156,157,69,162,35,37,167,40,41,42,43,44,45,174,175,176,178,179,53,54,57,187,188,191,193,194,196,197,72,73,74,76,77,80,81,163,169,222,224,225,226,227,228,229,232,233,234,235,236,237,238,239,240,241,116,117,118,119,123,170,126,127],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/chardet/universaldetector.py":[36,39,40,41,43,44,45,46,47,48,51,66,68,69,70,71,72,73,74,75,76,77,78,79,81,220,94,111],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/urllib3/util/retry.py":[1,2,3,4,5,6,7,9,394,279,273,142,144,145,147,20,150,23,24,153,154,155,285,158,159,160,162,27,294,167,168,169,170,171,257,173,175,172,310,189,190,319,320,203,401,217,152,166,17,233,157,243,251],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/chardet/latin1prober.py":[130,29,30,32,34,35,36,37,38,39,40,41,42,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,92,96,97,103,108,112,116],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/nose/result.py":[38,39,40,41,106,43,44,109,110,104,105,103],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/chardet/escprober.py":[35,69,40,73,42,77,83,58,28,29,30,31],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/chardet/jisfreq.py":[322,44,47],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/chardet/codingstatemachine.py":[33,66,86,80,83,54,55,28,30,63],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/urllib3/util/__init__.py":[1,3,4,5,6,16,21,22,28,53],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/yaml/scanner.py":[514,375,1149,1197,1177,120,27,29,542,1055,32,33,35,548,549,38,39,40,41,42,43,44,46,48,561,121,1204,60,181,64,67,70,1095,465,73,76,79,593,594,595,596,1109,598,478,609,1124,101,620,109,1137,115,116,117,118,631,1144,1145,634,191,1148,637,1150,1151,1152,311,130,132,1157,134,1159,648,1161,1162,651,1164,1165,1166,1167,1168,1169,1170,1171,1172,1173,1174,1175,1176,665,1178,156,1181,1182,1183,1184,1187,1189,166,113,680,169,1194,1195,1196,173,1198,687,177,773,30,695,654,703,192,195,708,199,200,204,782,1228,718,207,208,721,722,211,1230,365,728,219,220,122,223,227,550,1254,231,1201,235,749,125,979,247,1272,127,769,771,772,261,774,775,778,215,269,270,271,784,273,274,276,196,657,1156,283,284,285,286,555,803,292,163,146,1319,470,298,302,303,304,136,306,307,309,137,824,314,138,318,123,322,1163,1153,652,843,1356,551,337,338,142,857,346,143,144,867,356,1382,145,361,364,877,368,371,374,887,376,660,379,382,149,1408,385,387,901,422,151,312,399,402,1427,405,153,239,1436,1437,150,1443,421,1446,424,937,427,430,433,243,436,439,440,441,442,159,444,445,447,448,450,453,456,129,459,663,305,462,463,464,248,467,203,473,476,477,1158,479,481,1190,1191,272,1192,545,425,1193,170,341],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/chardet/sbcharsetprober.py":[33,34,35,36,37,70,39,77,53,124,29,30,63],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/nose/plugins/deprecated.py":[40,42,43,44],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/attr/_compat.py":[1,3,4,7,10,11,15,16,19,21,24,28,31,33,38,43,48,53,58,63,68,70,72,73,74,75],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/chardet/big5prober.py":[34,35,41,45,28,29,30,31],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/requests/compat.py":[9,11,13,20,23,26,28,29,30,33,39,40,43,44,45,46,47,49,51,52,53,54,55,56],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/chardet/gb2312freq.py":[281,42,44],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/requests/structures.py":[8,10,12,15,40,42,43,44,45,46,48,51,53,54,56,59,60,62,63,65,73,82,85,89,90,92,93,94,96,99,104],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/yaml/nodes.py":[2,3,8,25,26,28,29,30,31,32,33,35,37,38,39,40,41,42,44,45,47,48],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/attr/_funcs.py":[1,3,5,6,7,10,11,142,151,154,39,40,41,42,43,45,46,49,50,51,52,55,57,187,64,67,70,71],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/nose/plugins/debug.py":[40,41,42,43],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/chardet/langcyrillicmodel.py":[321,141,302,278,281,282,283,284,285,286,287,290,291,292,293,294,295,296,299,300,301,46,303,304,305,308,309,310,311,312,313,314,317,318,319,320,65,322,323,326,327,328,329,330,331,332,84,103,122],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/chardet/__init__.py":[24,19,20,21],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/chardet/jpcntx.py":[31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,116,117,118,119,120,121,123,131,143,170,173,180,183,184,188,192,212,213],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/chardet/mbcssm.py":[534,538,539,28,541,542,543,544,545,546,547,548,549,550,551,552,553,554,555,556,557,558,559,560,561,562,563,566,568,569,570,571,572,64,68,69,70,73,75,76,77,78,79,99,104,105,106,107,108,109,110,113,115,116,117,118,119,155,159,160,161,162,163,166,168,169,170,171,172,208,212,213,216,218,219,220,221,222,258,262,263,264,265,266,267,270,272,273,274,275,276,312,316,317,318,319,320,321,329,331,332,333,334,335,540,373,377,378,379,382,384,385,386,387,388,424,428,429,430,431,432,433,434,437,439,440,441,442,443,479,483,484,485,486,487,488,489,492,494,495,496,497,498],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/chardet/euckrfreq.py":[41,43,193],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/nose/plugins/errorclass.py":[148,150,151,152,153,154],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/chardet/euctwprober.py":[33,34,40,44,28,29,30,31],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/urllib3/connectionpool.py":[1,2,3,4,5,7,8,777,11,155,448,282,409,26,27,28,29,158,159,214,161,162,35,36,38,39,40,41,42,43,812,794,46,157,48,50,883,52,54,58,757,62,406,64,65,67,836,292,201,74,759,78,288,81,163,86,855,164,731,94,97,321,449,747,304,749,750,752,753,754,755,756,446,758,425,447,252],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/nose/plugins/base.py":[98,100,101,102],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/certifi/core.py":[35,9,10,11,14,18,21,22,24,27],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/nose/plugins/prof.py":[71,74,75,76,80,81,82,83,84,57],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/requests/hooks.py":[34,13,14,17,18,23,25,26,27],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/urllib3/exceptions.py":[1,2,8,9,10,13,14,15,18,19,20,24,29,30,31,35,40,41,42,45,46,47,50,51,52,55,56,57,61,66,74,76,85,86,88,94,95,96,99,104,105,108,109,110,115,116,117,120,121,122,125,126,127,130,131,132,135,136,137,140,141,143,150,151,152,153,156,157,158,161,162,163,166,167,168,171,172,173,176,177,178,181,182,183,186,190,191,194,195,196,199,203,204,207,214,215,218,223,224,225,228,229,232,237,238,239,244,245,246],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/urllib3/util/selectors.py":[256,130,131,133,8,9,10,11,12,13,14,16,17,18,19,21,22,150,24,25,26,539,29,30,287,288,289,34,164,37,294,389,41,534,278,172,302,136,308,286,312,58,543,266,565,192,139,71,456,457,458,459,206,463,336,466,270,165,226,484,146,245,505,250,281,127],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/yaml/emitter.py":[516,9,11,12,14,15,17,18,1047,540,31,33,34,35,38,39,922,559,1082,582,106,619,111,629,120,133,146,160,170,175,178,546,215,227,233,234,261,267,275,790,281,794,799,803,293,470,816,311,317,840,334,847,856,355,360,829,369,374,377,389,393,396,910,911,912,913,914,915,916,917,918,919,920,921,410,923,924,925,415,928,423,427,431,438,460,982,993,495],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/polling.py":[1,3,5,6,7,44,39,12,13,14,45,19,20,23,24,33,27],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/urllib3/response.py":[1,2,3,4,5,6,7,9,10,14,15,16,17,19,22,24,29,32,545,513,55,57,60,522,63,69,76,108,110,111,113,114,531,116,118,121,122,123,124,125,126,127,128,130,131,132,133,134,136,139,140,142,143,146,147,148,150,151,155,158,161,174,115,181,190,194,202,206,208,219,239,240,245,248,250,256,257,260,264,265,273,276,278,289,298,300,301,302,324,328,567,343,346,367,368,371,374,375,377,378,383,384,385,393,394,395,403,404,405,408,410,413,415,431,435,436,438,439,441,471,474,478,485,487,489,491,492,496,505,509],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/urllib3/util/ssl_.py":[1,2,3,4,6,7,9,215,12,13,14,15,18,19,20,21,278,279,280,25,38,39,42,43,44,45,176,50,51,70,71,72,73,74,75,76,77,78,79,80,81,82,83,86,87,216,199,281,149],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/urllib3/util/wait.py":[1,36,29,9],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/requests/exceptions.py":[8,9,12,15,17,19,20,21,22,23,24,25,28,29,32,33,36,37,40,41,44,50,53,57,60,61,64,65,68,69,72,73,76,77,80,81,84,85,88,89,92,93,96,97,100,101,104,105,110,111,112,115,116,117,120,121,122],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/attr/exceptions.py":[1,4,33,39,12,13,14,17,22,25,30],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/urllib3/connection.py":[1,2,3,4,5,6,7,8,9,10,11,13,14,15,282,259,23,25,26,27,28,154,31,37,39,169,257,48,50,52,54,55,56,62,65,66,67,70,258,260,208,209,211,213,214,215,95,97,354,228,101,165,104,106,255,368,370,371,246,250,251,252,253,254,127],"/Users/strucka/Projects/tes-python-packages/py-tes/tes/utils.py":[1,3,4,5,6,7,8,11,12,13,15,16,17,18,20,21,22,23,24,25,28,29,30,31,32,35,36,37,38,39,40,41,42,43,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,87,88,89,90,91,92,93],"/Users/strucka/Projects/tes-python-packages/py-tes/tes/client.py":[1,3,4,6,7,8,10,11,14,15,16,17,19,21,22,27,28,29,30,31,32,33,34,35,36,37,38,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/chardet/langbulgarianmodel.py":[224,225,226,227,72,209,212,53,214,215,216,217,218,223,221,222,213],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/nose/plugins/manager.py":[128,262,263,264,265,272,273,274,149,167,166,295,168,301,302,177,178,184,249,88,89,93,94,95,96,99,252,106,107,111,114,123,118,105,120,121,250,251,124,253,254],"/Users/strucka/Projects/tes-python-packages/py-tes/tests/test_models.py":[1,2,4,7,8,10,11,12,17,19,20,21,26,27,29,30,32,33,35,36,37,39,41,43],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/requests_mock/__init__.py":[32,33,34,35,36,13,14,15,16,17,20,21,22,23,24,25,26,27,28,30,31],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/requests/cookies.py":[512,514,515,516,520,10,523,12,13,14,15,17,18,20,21,535,536,537,26,542,36,38,39,40,41,43,46,49,52,66,69,72,75,79,82,83,85,89,93,98,529,103,105,110,112,113,115,119,532,126,127,128,136,142,143,144,147,166,169,172,188,190,202,219,228,236,245,253,262,271,279,287,300,316,322,331,338,344,349,351,352,357,377,402,409,415,422,437,472,503,511],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/nose/util.py":[320,446,278,263,520,521,266,267,140,397,270,271,272,273,403,660,405,662,663,408,409,410,411,312,393,163,164,406,470,321,264,306,307,308,309,310,265,184,313,187,188,189,190,319,192,279,195,407,471,311,337,338,339,340,398,342,343,399,479,323,481,483,484,485,486,449,404,448,318,502,503,504,505,506,447,276],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/nose/plugins/isolate.py":[61,62],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/chardet/langthaimodel.py":[192,193,194,195,196,197,198,52,189],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/nose/plugins/multiprocess.py":[224,225,226,227,231,233,234,235,238,223],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/nose/importer.py":[143,144,146,147,148,149,151,152,153,154,155,156,157,158,32,161,165,166,167,40,41,42,44,45,47,30,54,59,62,53,65,66,67,68,70,71,72,75,76,77,78,79,80,81,94,96,97,98,99,101,103,104,63],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/yaml/reader.py":[129,133,134,135,137,138,139,140,175,173,152,146,147,20,149,22,151,24,26,155,156,154,150,33,169,170,171,172,45,174,157,177,59,60,61,62,63,64,65,66,67,68,69,70,71,72,76,77,78,79,87,88,89,94,95,97,99,100,101,102,103,104,105,106,107,18,110,111,112,114,115,116,117,148,122,123,125,126],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/idna/idnadata.py":[3,37,54,65,71,81,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,288,289,290,291,292,293,294,295,296,297,298,299,300,301,302,303,304,305,306,307,308,309,310,311,312,313,314,315,316,317,318,319,320,321,322,323,324,325,326,327,328,329,330,331,332,333,334,335,336,337,338,339,340,341,342,343,344,345,346,347,348,349,350,351,352,353,354,355,356,357,358,359,360,361,362,363,364,365,366,367,368,369,370,371,372,373,374,375,376,377,378,379,380,381,382,383,384,385,386,387,388,389,390,391,392,393,394,395,396,397,398,399,400,401,402,403,404,405,406,407,408,409,410,411,412,413,414,415,416,417,418,419,420,421,422,423,424,425,426,427,428,429,430,431,432,433,434,435,436,437,438,439,440,441,442,443,444,445,446,447,448,449,450,451,452,453,454,455,456,457,458,459,460,461,462,463,464,465,466,467,468,469,470,471,472,473,474,475,476,477,478,479,480,481,482,483,484,485,486,487,488,489,490,491,492,493,494,495,496,497,498,499,500,501,502,503,504,505,506,507,508,509,510,511,512,513,514,515,516,517,518,519,520,521,522,523,524,525,526,527,528,529,530,531,532,533,534,535,536,537,538,539,540,541,542,543,544,545,546,547,548,549,550,551,552,553,554,555,556,557,558,559,560,561,562,563,564,565,566,567,568,569,570,571,572,573,574,575,576,577,578,579,580,581,582,583,584,585,586,587,588,590,1571,1574,1582],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/pkg_resources/extern/__init__.py":[28,29,30],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/chardet/eucjpprober.py":[32,33,36,37,44,48,52,56,89,28,29,30,31],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/chardet/chardistribution.py":[171,132,133,139,151,152,28,30,32,34,36,40,41,42,43,44,46,177,158,61,192,193,70,199,84,217,218,224,100,105,113,114,120,170],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/yaml/dumper.py":[2,4,5,6,7,9,12,13,14,15,16,27,30,31,32,33,34,45,48,49,50,51,52],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/requests/certs.py":[17,14,15],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/urllib3/util/url.py":[1,2,201,4,7,11,14,19,20,22,23,24,132,154,27,28,26,158,159,160,33,162,163,164,38,167,168,176,172,174,29,48,179,30,55,184,31,189,190,192,193,195,198,161,200,129,211,215,219,222,95,225,99,115,116,117,118,119,120,122,123,124,126],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/urllib3/util/connection.py":[96,1,2,3,4,37,7,107,130,110,109,112,120,121,118,119,88,36,125,126,127],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/requests/__version__.py":[5,6,7,8,9,10,11,12,13,14],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/urllib3/util/response.py":[1,2,4,69,38,7,15,18,19,20,22,24],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/requests/_internal_utils.py":[37,38,39,40,9,11,14,19,20,27,30],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/yaml/tokens.py":[2,3,4,5,6,17,18,19,25,26,28,29,31,32,33,34,35,36,37,39,40,42,43,45,46,48,49,51,52,54,55,57,58,60,61,63,64,66,67,69,70,72,73,75,76,77,82,83,84,89,90,91,96,97,98,99,100,101,102,103],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/attr/filters.py":[3,5,38,7,8,11,21],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/chardet/big5freq.py":[384,43,46],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/yaml/loader.py":[33,2,4,5,6,7,8,9,11,13,21,23,24,25,26,27,28,29,31],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/yaml/parser.py":[512,526,537,538,539,540,542,543,544,545,546,551,552,553,554,555,556,563,564,565,566,567,569,570,571,572,573,62,64,65,66,67,69,70,583,72,587,76,77,78,81,82,83,84,85,86,87,89,91,92,94,96,97,98,99,100,102,103,104,105,107,109,112,114,116,117,118,119,120,121,127,130,131,132,135,137,139,142,143,144,145,146,147,148,151,152,154,159,162,166,183,184,185,186,187,188,190,193,194,195,196,200,201,204,206,208,217,264,265,267,268,270,273,274,279,280,281,282,292,301,315,316,317,318,319,325,326,327,328,330,331,334,335,336,337,338,339,340,341,342,343,344,345,346,372,574,376,381,402,422,427,446,471,472,473,474,476,477,478,479,480,486,493,494,495,496,497,498,499,500,502],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/requests_mock/request.py":[130,134,136,138,139,13,14,143,16,17,18,21,26,28,29,30,31,32,36,37,38,39,40,44,46,47,49,51,52,54,55,57,59,61,63,65,67,69,76,141,95,97,99,101,103,146,110,112,114,118,122,126],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/urllib3/filepost.py":[1,2,4,5,7,8,9,11,14,21,41,59],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/nose/plugins/doctests.py":[192,193,194,195,188,189,190,191],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/requests/status_codes.py":[3,5,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,25,26,27,28,29,30,31,32,34,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,72,73,74,75,76,77,78,79,80,81,82,85,87,88,89,90,91],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/requests_mock/mocker.py":[68,128,134,137,138,87,140,13,15,144,17,18,20,21,22,23,24,25,26,155,28,158,31,161,36,165,38,39,40,41,42,43,172,45,180,181,183,184,244,187,188,190,63,64,65,67,196,70,73,205,78,141,81,83,84,86,185,88,152,143,223,123,100,101,167,146,110,114,116,170,153,121,122,127,125,126,149],"/Users/strucka/Projects/tes-python-packages/venv/lib/python2.7/site-packages/chardet/escsm.py":[129,131,132,133,134,135,136,28,243,170,174,175,176,177,178,179,180,181,182,185,187,188,189,62,191,192,66,67,68,69,70,71,74,76,77,78,79,80,81,226,230,231,232,233,234,237,239,240,241,242,115,244,190,119,120,121,122,123,124,125,126],"/home/kellrott/workspaces/py-tes/tes/__init__.py":[1,3,4,5,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,22,41],"/home/kellrott/workspaces/py-tes/tes/client.py":[1,3,4,5,7,8,9,10,12,15,18,22,23,24,25,26,27,28,29,30,31,32,33,34,36,37,45,53,67,77,86,103,123,19,38,39,78,79,124,126,128,130,134,135,137,80,81,82,83,84,40,41,42,54,55,59,127,60,61,62,64,65,46,47,48,49,50,51,68,69,70,129,71,72,73,74,75,87,88,89,90,91,92,94,96,97,98,99,100,101,104,107,110,111,115,116,105,119,121,120,117],"/home/kellrott/workspaces/py-tes/tes/models.py":[1,3,4,5,6,8,9,10,11,14,15,16,18,33,40,44,56,70,76,82,88,89,91,97,104,105,106,107,109,110,112,113,115,116,118,119,121,122,126,127,128,129,131,132,134,135,137,138,140,141,145,146,147,148,150,151,153,154,156,157,159,160,41,164,165,166,167,169,170,172,173,175,176,178,179,181,182,184,185,189,190,191,192,193,194,196,197,198,199,201,202,204,205,207,208,212,213,214,215,217,218,220,221,225,226,227,228,229,230,232,233,234,235,237,238,240,241,243,244,246,247,251,252,253,254,256,257,258,259,260,263,264,266,267,269,270,272,273,275,276,278,279,281,282,284,285,287,288,290,291,292,293,296,361,362,363,364,366,367,371,372,373,374,378,379,380,383,384,385,386,388,389,391,392,396,397,398,399,403,404,405,408,409,410,411,413,414,416,417,419,420,422,423,427,428,429,430,432,433,57,62,63,58,59,65,22,77,79,98,99,92,93,94,45,47,48,50,49,53,46,100,297,298,301,302,304,306,309,312,315,322,333,342,348,355,358,323,324,325,326,328,330,356,334,335,337,339,23,24,26,27,28,29,30,78,71,72],"/home/kellrott/workspaces/py-tes/tes/utils.py":[1,3,4,6,7,11,12,15,20,21,25,26,30,90,91,92,93,94,95,96,31,33,34,38,39,40,41,16,17,46,47,48,49,50,51,45,54,63,64,65,66,77,79,80,87,67,75,76,55,56,57,61,27,81,82,83,84,85,22,32,58,68,69,70,71,72,73]}} \ No newline at end of file diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000..26c23f9 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,36 @@ +name: Building & publishing docs + +on: + push: + branches: [master,api-docs] + +jobs: + docs: + runs-on: ubuntu-latest + + steps: + - name: Check out code + uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.11" + + - name: Install requirements + run: pip install -r docs/requirements.txt + + - name: Install app + run: pip install . + + - name: Create markdown documents + run: | + lazydocs \ + --output-path="./docs/docstrings" \ + --overview-file="README.md" \ + --src-base-url="https://github.com/ohsu-comp-bio/py-tes/blob/master/" \ + --validate \ + tes + + - name: Build docs + run: mkdocs build diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 68873ce..8425a66 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,26 +1,47 @@ -name: py-test_file +name: Linting and testing -on: [ pull_request ] +on: [pull_request] jobs: - test: runs-on: ubuntu-latest - steps: - - name: Check out code - uses: actions/checkout@v2 - - - name: Requirements - run: pip install -r requirements.txt - - name: Test Requirements - run: pip install -r tests/requirements.txt + strategy: + fail-fast: false + matrix: + version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] - - name: Install - run: python setup.py install --user - - - name: Flake - run: flake8 . - - - name: Test - run: python -m nose tests --with-coverage --cover-package tes --cover-min-percentage 80 + steps: + - name: Check out code + uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.version }} + + - name: Install requirements + run: | + pip install -r requirements.txt + pip install -r tests/requirements.txt + + - name: Install app + run: pip install . + + - name: Lint with Flake8 + run: flake8 --max-line-length=120 . + + - name: Run unit tests + run: | + pytest \ + --cov=tes/ \ + --cov-branch \ + --cov-report=term-missing \ + --cov-fail-under=99 \ + --ignore=tests/integration + + - name: Run integration tests + run: | + /bin/bash -c "$(curl -fsSL https://github.com/ohsu-comp-bio/funnel/releases/download/0.11.0-rc.5/install.sh)" -- 0.11.0-rc.5 + funnel server --LocalStorage.AllowedDirs $HOME run & + pytest tests/integration diff --git a/.gitignore b/.gitignore index d22d113..2623257 100644 --- a/.gitignore +++ b/.gitignore @@ -16,5 +16,181 @@ eggs/ \#*\# .desktop +# Misc test_tmp -*venv* \ No newline at end of file +*venv* +docs/docstrings + +# Created by https://www.toptal.com/developers/gitignore/api/python +# Edit at https://www.toptal.com/developers/gitignore?templates=python + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +### Python Patch ### +# Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration +poetry.toml + +# ruff +.ruff_cache/ + +# End of https://www.toptal.com/developers/gitignore/api/python diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 9623fa9..0000000 --- a/.travis.yml +++ /dev/null @@ -1,22 +0,0 @@ -language: python -python: - - 2.7 - - 3.5 - - 3.6 - -install: - - pip install pip --upgrade - - pip install setuptools --upgrade - - pip install -r tests/requirements.txt - - pip install -r requirements.txt - - python setup.py install - -script: - - flake8 . - - python -m nose tests --with-coverage --cover-package tes --cover-min-percentage 80 - -after_success: - - coveralls - -notifications: - email: false diff --git a/README.md b/README.md index 62110a7..d30de08 100644 --- a/README.md +++ b/README.md @@ -1,26 +1,23 @@ -[![Build Status](https://travis-ci.org/ohsu-comp-bio/py-tes.svg?branch=master)](https://travis-ci.org/ohsu-comp-bio/py-tes) -[![Coverage Status](https://coveralls.io/repos/github/ohsu-comp-bio/py-tes/badge.svg?branch=master)](https://coveralls.io/github/ohsu-comp-bio/py-tes?branch=master) -[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) +# py-tes -py-tes -====== +[![GitHub Actions Test Status](https://img.shields.io/github/actions/workflow/status/ohsu-comp-bio/py-tes/tests.yml?logo=github)](https://github.com/ohsu-comp-bio/py-tes/actions) [![image](https://coveralls.io/repos/github/ohsu-comp-bio/py-tes/badge.svg?branch=master)](https://coveralls.io/github/ohsu-comp-bio/py-tes?branch=master) [![image](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) -_py-tes_ is a library for interacting with servers implementing the [GA4GH Task Execution Schema](https://github.com/ga4gh/task-execution-schemas). +*py-tes* is a library for interacting with servers implementing the +[GA4GH Task Execution +Schema](https://github.com/ga4gh/task-execution-schemas). - -### Install +## Install Available on [PyPI](https://pypi.org/project/py-tes/). -``` -pip install py-tes -``` + pip install py-tes -### Example +## Example -``` +``` python import tes +# define task task = tes.Task( executors=[ tes.Executor( @@ -30,8 +27,135 @@ task = tes.Task( ] ) -cli = tes.HTTPClient("http://funnel.example.com", timeout=5) +# create client +cli = tes.HTTPClient("https://funnel.example.com", timeout=5) + +# access endpoints +service_info = cli.get_service_info() task_id = cli.create_task(task) -res = cli.get_task(task_id) +task_info = cli.get_task(task_id, view="BASIC") cli.cancel_task(task_id) +tasks_list = cli.list_tasks(view="MINIMAL") # default view +``` + +## How to... + +> Makes use of the objects above... + +### ...export a model to a dictionary + +``` python +task_dict = task.as_dict(drop_empty=False) +``` + +`task_dict` contents: + +``` console +{'id': None, 'state': None, 'name': None, 'description': None, 'inputs': None, 'outputs': None, 'resources': None, 'executors': [{'image': 'alpine', 'command': ['echo', 'hello'], 'workdir': None, 'stdin': None, 'stdout': None, 'stderr': None, 'env': None}], 'volumes': None, 'tags': None, 'logs': None, 'creation_time': None} +``` + +### ...export a model to JSON + +``` python +task_json = task.as_json() # also accepts `drop_empty` arg +``` + +`task_json` contents: + +``` console +{"executors": [{"image": "alpine", "command": ["echo", "hello"]}]} +``` + +### ...pretty print a model + +``` python +print(task.as_json(indent=3)) # keyword args are passed to `json.dumps()` +``` + +Output: + +``` json +{ + "executors": [ + { + "image": "alpine", + "command": [ + "echo", + "hello" + ] + } + ] +} +``` + +### ...access a specific task from the task list + +``` python +specific_task = tasks_list.tasks[5] ``` + +`specific_task` contents: + +``` console +Task(id='393K43', state='COMPLETE', name=None, description=None, inputs=None, outputs=None, resources=None, executors=None, volumes=None, tags=None, logs=None, creation_time=None) +``` + +### ...iterate over task list items + +``` python +for t in tasks_list[:3]: + print(t.as_json(indent=3)) +``` + +Output: + +``` console +{ + "id": "task_A2GFS4", + "state": "RUNNING" +} +{ + "id": "task_O8G1PZ", + "state": "CANCELED" +} +{ + "id": "task_W246I6", + "state": "COMPLETE" +} +``` + +### ...instantiate a model from a JSON representation + +``` python +task_from_json = tes.client.unmarshal(task_json, tes.Task) +``` + +`task_from_json` contents: + +``` console +Task(id=None, state=None, name=None, description=None, inputs=None, outputs=None, resources=None, executors=[Executor(image='alpine', command=['echo', 'hello'], workdir=None, stdin=None, stdout=None, stderr=None, env=None)], volumes=None, tags=None, logs=None, creation_time=None) +``` + +Which is equivalent to `task`: + +``` python +print(task_from_json == task) +``` + +Output: + +``` console +True +``` + +## Resources + +- [ga4gh-tes](https://github.com/microsoft/ga4gh-tes) : C# implementation of the GA4GH TES API; provides distributed batch task execution on Microsoft Azure + +- [cwl-tes](https://github.com/ohsu-comp-bio/cwl-tes) : cwl-tes submits your tasks to a TES server. Task submission is parallelized when possible. + +- [Funnel](https://ohsu-comp-bio.github.io/funnel/): Funnel is a toolkit for distributed task execution with a simple API. + +- [Snakemake](https://snakemake.github.io/) : The Snakemape workflow management system is a tool to create reproducible and scalable data analyses + +- [Nextflow](https://www.nextflow.io/): Nextflow enables scalable and reproducible scientific workflows using software containers. It allows the adaptation of pipelines written in the most common scripting languages. diff --git a/docs/.pages b/docs/.pages new file mode 100644 index 0000000..e8b6473 --- /dev/null +++ b/docs/.pages @@ -0,0 +1,3 @@ +nav: + - Overview: README.md + - ... diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..f494a3f --- /dev/null +++ b/docs/README.md @@ -0,0 +1,149 @@ +py-tes +====== + +_py-tes_ is a library for interacting with servers implementing the [GA4GH Task Execution Schema](https://github.com/ga4gh/task-execution-schemas). + + +### Install + +Available on [PyPI](https://pypi.org/project/py-tes/). + +``` +pip install py-tes +``` + +### Example + +```python +import tes + +# define task +task = tes.Task( + executors=[ + tes.Executor( + image="alpine", + command=["echo", "hello"] + ) + ] +) + +# create client +cli = tes.HTTPClient("https://funnel.example.com", timeout=5) + +# access endpoints +service_info = cli.get_service_info() +task_id = cli.create_task(task) +task_info = cli.get_task(task_id, view="BASIC") +cli.cancel_task(task_id) +tasks_list = cli.list_tasks(view="MINIMAL") # default view +``` + +### How to... + +> Makes use of the objects above... + +#### ...export a model to a dictionary + +```python +task_dict = task.as_dict(drop_empty=False) +``` + +`task_dict` contents: + +```console +{'id': None, 'state': None, 'name': None, 'description': None, 'inputs': None, 'outputs': None, 'resources': None, 'executors': [{'image': 'alpine', 'command': ['echo', 'hello'], 'workdir': None, 'stdin': None, 'stdout': None, 'stderr': None, 'env': None}], 'volumes': None, 'tags': None, 'logs': None, 'creation_time': None} +``` + +#### ...export a model to JSON + +```python +task_json = task.as_json() # also accepts `drop_empty` arg +``` + +`task_json` contents: + +```console +{"executors": [{"image": "alpine", "command": ["echo", "hello"]}]} +``` + +#### ...pretty print a model + +```python +print(task.as_json(indent=3)) # keyword args are passed to `json.dumps()` +``` + +Output: + +```json +{ + "executors": [ + { + "image": "alpine", + "command": [ + "echo", + "hello" + ] + } + ] +} +``` + +#### ...access a specific task from the task list + +```python +specific_task = tasks_list.tasks[5] +``` + +`specific_task` contents: + +```console +Task(id='393K43', state='COMPLETE', name=None, description=None, inputs=None, outputs=None, resources=None, executors=None, volumes=None, tags=None, logs=None, creation_time=None) +``` + +#### ...iterate over task list items + +```python +for t in tasks_list[:3]: + print(t.as_json(indent=3)) +``` + +Output: + +```console +{ + "id": "task_A2GFS4", + "state": "RUNNING" +} +{ + "id": "task_O8G1PZ", + "state": "CANCELED" +} +{ + "id": "task_W246I6", + "state": "COMPLETE" +} +``` + +#### ...instantiate a model from a JSON representation + +```python +task_from_json = tes.client.unmarshal(task_json, tes.Task) +``` + +`task_from_json` contents: + +```console +Task(id=None, state=None, name=None, description=None, inputs=None, outputs=None, resources=None, executors=[Executor(image='alpine', command=['echo', 'hello'], workdir=None, stdin=None, stdout=None, stderr=None, env=None)], volumes=None, tags=None, logs=None, creation_time=None) +``` + +Which is equivalent to `task`: + +```python +print(task_from_json == task) +``` + +Output: + +```console +True +``` diff --git a/docs/docstring/.pages b/docs/docstring/.pages new file mode 100644 index 0000000..db48efa --- /dev/null +++ b/docs/docstring/.pages @@ -0,0 +1,4 @@ +title: API Reference +nav: + - Overview: README.md + - ... diff --git a/docs/docstring/README.md b/docs/docstring/README.md new file mode 100644 index 0000000..9c86184 --- /dev/null +++ b/docs/docstring/README.md @@ -0,0 +1,48 @@ + + +# API Overview + +## Modules + +- [`client`](./client.md#module-client): TES access methods and helper functions. +- [`models`](./models.md#module-models): TES models, converters, validators and helpers. +- [`utils`](./utils.md#module-utils): Exceptions and utilities. + +## Classes + +- [`client.HTTPClient`](./client.md#class-httpclient): HTTP client class for interacting with the TES API. +- [`models.Base`](./models.md#class-base): `attrs` base class for all TES and helper models. +- [`models.CancelTaskRequest`](./models.md#class-canceltaskrequest): `attrs` model class for `POST /tasks/{id}:cancel` request parameters. +- [`models.CancelTaskResponse`](./models.md#class-canceltaskresponse): TES `tesCancelTaskResponse` `attrs` model class. +- [`models.CreateTaskResponse`](./models.md#class-createtaskresponse): TES `tesCreateTaskResponse` `attrs` model class. +- [`models.Executor`](./models.md#class-executor): TES `tesExecutor` `attrs` model class. +- [`models.ExecutorLog`](./models.md#class-executorlog): TES `tesExecutorLog` `attrs` model class. +- [`models.GetTaskRequest`](./models.md#class-gettaskrequest): `attrs` model class for `GET /tasks/{id}` request parameters. +- [`models.Input`](./models.md#class-input): TES `tesInput` `attrs` model class. +- [`models.ListTasksRequest`](./models.md#class-listtasksrequest): `attrs` model class for `GET /tasks` request parameters. +- [`models.ListTasksResponse`](./models.md#class-listtasksresponse): TES `tesListTasksResponse` `attrs` model class. +- [`models.Output`](./models.md#class-output): TES `tesOutput` `attrs` model class. +- [`models.OutputFileLog`](./models.md#class-outputfilelog): TES `tesOutputFileLog` `attrs` model class. +- [`models.Resources`](./models.md#class-resources): TES `tesResources` `attrs` model class. +- [`models.ServiceInfo`](./models.md#class-serviceinfo): TES `tesServiceInfo` `attrs` model class. +- [`models.ServiceInfoRequest`](./models.md#class-serviceinforequest): `attrs` model class for `GET /service-info` request parameters. +- [`models.Task`](./models.md#class-task): TES `tesTask` `attrs` model class. +- [`models.TaskLog`](./models.md#class-tasklog): TES `tesTaskLog` `attrs` model class. +- [`utils.TimeoutError`](./utils.md#class-timeouterror) +- [`utils.UnmarshalError`](./utils.md#class-unmarshalerror): Raised when a JSON string cannot be unmarshalled to a TES model. + +## Functions + +- [`client.process_url`](./client.md#function-process_url) +- [`models.datetime_json_handler`](./models.md#function-datetime_json_handler): JSON handler for `datetime` objects. +- [`models.int64conv`](./models.md#function-int64conv): Convert string to `int64`. +- [`models.list_of`](./models.md#function-list_of): `attrs` validator for lists of a given type. +- [`models.strconv`](./models.md#function-strconv): Explicitly cast a string-like value or list thereof to string(s). +- [`models.timestampconv`](./models.md#function-timestampconv): Convert string to `datetime`. +- [`utils.camel_to_snake`](./utils.md#function-camel_to_snake): Converts camelCase to snake_case. +- [`utils.unmarshal`](./utils.md#function-unmarshal): Unmarshal a JSON string to a TES model. + + +--- + +_This file was automatically generated via [lazydocs](https://github.com/ml-tooling/lazydocs)._ diff --git a/docs/docstring/client.md b/docs/docstring/client.md new file mode 100644 index 0000000..b6cd3d8 --- /dev/null +++ b/docs/docstring/client.md @@ -0,0 +1,186 @@ + + + + +# module `client` +TES access methods and helper functions. + + +--- + + + +## function `process_url` + +```python +process_url(value) +``` + + + + + + +--- + + + +## class `HTTPClient` +HTTP client class for interacting with the TES API. + + + +### method `__init__` + +```python +__init__( + url, + timeout: int = 10, + user: Any = None, + password: Any = None, + token: Any = None +) → None +``` + +Method generated by attrs for class HTTPClient. + + + + +--- + + + +### method `cancel_task` + +```python +cancel_task(task_id: str) → None +``` + +Access method for `POST /tasks/{id}:cancel`. + + + +**Args:** + + - `task_id`: TES Task ID. + +--- + + + +### method `create_task` + +```python +create_task(task: Task) → CreateTaskResponse +``` + +Access method for `POST /tasks`. + + + +**Args:** + + - `task`: `tes.models.Task` instance. + + + +**Returns:** + `tes.models.CreateTaskResponse` instance. + + + +**Raises:** + + - `TypeError`: If `task` is not a `tes.models.Task` instance. + +--- + + + +### method `get_service_info` + +```python +get_service_info() → ServiceInfo +``` + +Access method for `GET /service-info`. + + + +**Returns:** + `tes.models.ServiceInfo` instance. + +--- + + + +### method `get_task` + +```python +get_task(task_id: str, view: str = 'BASIC') → Task +``` + +Access method for `GET /tasks/{id}`. + + + +**Args:** + + - `task_id`: TES Task ID. + - `view`: Task info verbosity. One of `MINIMAL`, `BASIC` and `FULL`. + + + +**Returns:** + `tes.models.Task` instance. + +--- + + + +### method `list_tasks` + +```python +list_tasks( + view: str = 'MINIMAL', + page_size: Optional[int] = None, + page_token: Optional[str] = None +) → ListTasksResponse +``` + +Access method for `GET /tasks`. + + + +**Args:** + + - `view`: Task info verbosity. One of `MINIMAL`, `BASIC` and `FULL`. + - `page_size`: Number of tasks to return. + - `page_token`: Token to retrieve the next page of tasks. + + + +**Returns:** + `tes.models.ListTasksResponse` instance. + +--- + + + +### method `wait` + +```python +wait(task_id: str, timeout=None) → Task +``` + + + + + + + + +--- + +_This file was automatically generated via [lazydocs](https://github.com/ml-tooling/lazydocs)._ diff --git a/docs/docstring/models.md b/docs/docstring/models.md new file mode 100644 index 0000000..05cc326 --- /dev/null +++ b/docs/docstring/models.md @@ -0,0 +1,1052 @@ + + + + +# module `models` +TES models, converters, validators and helpers. + + +--- + + + +## function `list_of` + +```python +list_of(_type: Any) → _ListOfValidator +``` + +`attrs` validator for lists of a given type. + + + +**Args:** + + - `_type`: Type to validate. + + + +**Returns:** + `attrs` validator for the given type. + + +--- + + + +## function `strconv` + +```python +strconv(value: Any) → Any +``` + +Explicitly cast a string-like value or list thereof to string(s). + + + +**Args:** + + - `value`: Value to convert. + + + +**Returns:** + Converted value. If `value` is a list, all elements are converted to strings. If `value` is not string-like, it will be returned as is. + + +--- + + + +## function `int64conv` + +```python +int64conv(value: Optional[str]) → Optional[int] +``` + +Convert string to `int64`. + + + +**Args:** + + - `value`: String to convert. + + + +**Returns:** + Converted value. + + +--- + + + +## function `timestampconv` + +```python +timestampconv(value: Optional[str]) → Optional[datetime] +``` + +Convert string to `datetime`. + + + +**Args:** + + - `value`: String to convert. + + + +**Returns:** + Converted value. + + +--- + + + +## function `datetime_json_handler` + +```python +datetime_json_handler(x: Any) → str +``` + +JSON handler for `datetime` objects. + + + +**Args:** + + - `x`: Object to convert. + + + +**Returns:** + Converted object. + + + +**Raises:** + + - `TypeError`: If `x` is not a `datetime` object. + + +--- + + + +## class `Base` +`attrs` base class for all TES and helper models. + + + +### method `__init__` + +```python +__init__() → None +``` + +Method generated by attrs for class Base. + + + + +--- + + + +### method `as_dict` + +```python +as_dict(drop_empty: bool = True) → Dict[str, Any] +``` + + + + + +--- + + + +### method `as_json` + +```python +as_json(drop_empty: bool = True, **kwargs) → str +``` + + + + + + +--- + + + +## class `Input` +TES `tesInput` `attrs` model class. + + + +### method `__init__` + +```python +__init__( + url: Any = None, + path: Any = None, + type: str = 'FILE', + name: Any = None, + description: Any = None, + content: Any = None +) → None +``` + +Method generated by attrs for class Input. + + + + +--- + + + +### method `as_dict` + +```python +as_dict(drop_empty: bool = True) → Dict[str, Any] +``` + + + + + +--- + + + +### method `as_json` + +```python +as_json(drop_empty: bool = True, **kwargs) → str +``` + + + + + + +--- + + + +## class `Output` +TES `tesOutput` `attrs` model class. + + + +### method `__init__` + +```python +__init__( + url: Any = None, + path: Any = None, + type: str = 'FILE', + name: Any = None, + description: Any = None +) → None +``` + +Method generated by attrs for class Output. + + + + +--- + + + +### method `as_dict` + +```python +as_dict(drop_empty: bool = True) → Dict[str, Any] +``` + + + + + +--- + + + +### method `as_json` + +```python +as_json(drop_empty: bool = True, **kwargs) → str +``` + + + + + + +--- + + + +## class `Resources` +TES `tesResources` `attrs` model class. + + + +### method `__init__` + +```python +__init__( + cpu_cores: Optional[int] = None, + ram_gb: Optional[float, int] = None, + disk_gb: Optional[float, int] = None, + preemptible: Optional[bool] = None, + zones: Any = None +) → None +``` + +Method generated by attrs for class Resources. + + + + +--- + + + +### method `as_dict` + +```python +as_dict(drop_empty: bool = True) → Dict[str, Any] +``` + + + + + +--- + + + +### method `as_json` + +```python +as_json(drop_empty: bool = True, **kwargs) → str +``` + + + + + + +--- + + + +## class `Executor` +TES `tesExecutor` `attrs` model class. + + + +### method `__init__` + +```python +__init__( + image: Any, + command: Any, + workdir: Any = None, + stdin: Any = None, + stdout: Any = None, + stderr: Any = None, + env: Optional[Dict] = None +) → None +``` + +Method generated by attrs for class Executor. + + + + +--- + + + +### method `as_dict` + +```python +as_dict(drop_empty: bool = True) → Dict[str, Any] +``` + + + + + +--- + + + +### method `as_json` + +```python +as_json(drop_empty: bool = True, **kwargs) → str +``` + + + + + + +--- + + + +## class `ExecutorLog` +TES `tesExecutorLog` `attrs` model class. + + + +### method `__init__` + +```python +__init__( + start_time: Optional[str] = None, + end_time: Optional[str] = None, + stdout: Any = None, + stderr: Any = None, + exit_code: Optional[int] = None +) → None +``` + +Method generated by attrs for class ExecutorLog. + + + + +--- + + + +### method `as_dict` + +```python +as_dict(drop_empty: bool = True) → Dict[str, Any] +``` + + + + + +--- + + + +### method `as_json` + +```python +as_json(drop_empty: bool = True, **kwargs) → str +``` + + + + + + +--- + + + +## class `OutputFileLog` +TES `tesOutputFileLog` `attrs` model class. + + + +### method `__init__` + +```python +__init__( + url: Any = None, + path: Any = None, + size_bytes: Optional[str] = None +) → None +``` + +Method generated by attrs for class OutputFileLog. + + + + +--- + + + +### method `as_dict` + +```python +as_dict(drop_empty: bool = True) → Dict[str, Any] +``` + + + + + +--- + + + +### method `as_json` + +```python +as_json(drop_empty: bool = True, **kwargs) → str +``` + + + + + + +--- + + + +## class `TaskLog` +TES `tesTaskLog` `attrs` model class. + + + +### method `__init__` + +```python +__init__( + start_time: Optional[str] = None, + end_time: Optional[str] = None, + metadata: Optional[Dict] = None, + logs: Optional[List[ExecutorLog]] = None, + outputs: Optional[List[OutputFileLog]] = None, + system_logs: Optional[List[str]] = None +) → None +``` + +Method generated by attrs for class TaskLog. + + + + +--- + + + +### method `as_dict` + +```python +as_dict(drop_empty: bool = True) → Dict[str, Any] +``` + + + + + +--- + + + +### method `as_json` + +```python +as_json(drop_empty: bool = True, **kwargs) → str +``` + + + + + + +--- + + + +## class `Task` +TES `tesTask` `attrs` model class. + + + +### method `__init__` + +```python +__init__( + id: Any = None, + state: Optional[str] = None, + name: Any = None, + description: Any = None, + inputs: Optional[List[Input]] = None, + outputs: Optional[List[Output]] = None, + resources: Optional[Resources] = None, + executors: Optional[List[Executor]] = None, + volumes: Optional[List[str]] = None, + tags: Optional[Dict] = None, + logs: Optional[List[TaskLog]] = None, + creation_time: Optional[str] = None +) → None +``` + +Method generated by attrs for class Task. + + + + +--- + + + +### method `as_dict` + +```python +as_dict(drop_empty: bool = True) → Dict[str, Any] +``` + + + + + +--- + + + +### method `as_json` + +```python +as_json(drop_empty: bool = True, **kwargs) → str +``` + + + + + +--- + + + +### method `is_valid` + +```python +is_valid() → Tuple[bool, Optional[TypeError]] +``` + +Validate a `Task` model instance. + + + +**Returns:** + A tuple containing a boolean indicating whether the model is valid, and a `TypeError` if the model is invalid, or `None` if it is. + + +--- + + + +## class `GetTaskRequest` +`attrs` model class for `GET /tasks/{id}` request parameters. + + + +### method `__init__` + +```python +__init__(id: Any, view: Optional[str] = None) → None +``` + +Method generated by attrs for class GetTaskRequest. + + + + +--- + + + +### method `as_dict` + +```python +as_dict(drop_empty: bool = True) → Dict[str, Any] +``` + + + + + +--- + + + +### method `as_json` + +```python +as_json(drop_empty: bool = True, **kwargs) → str +``` + + + + + + +--- + + + +## class `CreateTaskResponse` +TES `tesCreateTaskResponse` `attrs` model class. + + + +### method `__init__` + +```python +__init__(id: Any) → None +``` + +Method generated by attrs for class CreateTaskResponse. + + + + +--- + + + +### method `as_dict` + +```python +as_dict(drop_empty: bool = True) → Dict[str, Any] +``` + + + + + +--- + + + +### method `as_json` + +```python +as_json(drop_empty: bool = True, **kwargs) → str +``` + + + + + + +--- + + + +## class `ServiceInfoRequest` +`attrs` model class for `GET /service-info` request parameters. + + + +### method `__init__` + +```python +__init__() → None +``` + +Method generated by attrs for class ServiceInfoRequest. + + + + +--- + + + +### method `as_dict` + +```python +as_dict(drop_empty: bool = True) → Dict[str, Any] +``` + + + + + +--- + + + +### method `as_json` + +```python +as_json(drop_empty: bool = True, **kwargs) → str +``` + + + + + + +--- + + + +## class `ServiceInfo` +TES `tesServiceInfo` `attrs` model class. + + + +### method `__init__` + +```python +__init__(name: Any = None, doc: Any = None, storage: Any = None) → None +``` + +Method generated by attrs for class ServiceInfo. + + + + +--- + + + +### method `as_dict` + +```python +as_dict(drop_empty: bool = True) → Dict[str, Any] +``` + + + + + +--- + + + +### method `as_json` + +```python +as_json(drop_empty: bool = True, **kwargs) → str +``` + + + + + + +--- + + + +## class `CancelTaskRequest` +`attrs` model class for `POST /tasks/{id}:cancel` request parameters. + + + +### method `__init__` + +```python +__init__(id: Any) → None +``` + +Method generated by attrs for class CancelTaskRequest. + + + + +--- + + + +### method `as_dict` + +```python +as_dict(drop_empty: bool = True) → Dict[str, Any] +``` + + + + + +--- + + + +### method `as_json` + +```python +as_json(drop_empty: bool = True, **kwargs) → str +``` + + + + + + +--- + + + +## class `CancelTaskResponse` +TES `tesCancelTaskResponse` `attrs` model class. + + + +### method `__init__` + +```python +__init__() → None +``` + +Method generated by attrs for class CancelTaskResponse. + + + + +--- + + + +### method `as_dict` + +```python +as_dict(drop_empty: bool = True) → Dict[str, Any] +``` + + + + + +--- + + + +### method `as_json` + +```python +as_json(drop_empty: bool = True, **kwargs) → str +``` + + + + + + +--- + + + +## class `ListTasksRequest` +`attrs` model class for `GET /tasks` request parameters. + + + +### method `__init__` + +```python +__init__( + project: Any = None, + name_prefix: Any = None, + page_size: Optional[int] = None, + page_token: Any = None, + view: Optional[str] = None +) → None +``` + +Method generated by attrs for class ListTasksRequest. + + + + +--- + + + +### method `as_dict` + +```python +as_dict(drop_empty: bool = True) → Dict[str, Any] +``` + + + + + +--- + + + +### method `as_json` + +```python +as_json(drop_empty: bool = True, **kwargs) → str +``` + + + + + + +--- + + + +## class `ListTasksResponse` +TES `tesListTasksResponse` `attrs` model class. + + + +### method `__init__` + +```python +__init__(tasks: Optional[List[Task]] = None, next_page_token: Any = None) → None +``` + +Method generated by attrs for class ListTasksResponse. + + + + +--- + + + +### method `as_dict` + +```python +as_dict(drop_empty: bool = True) → Dict[str, Any] +``` + + + + + +--- + + + +### method `as_json` + +```python +as_json(drop_empty: bool = True, **kwargs) → str +``` + + + + + + + + +--- + +_This file was automatically generated via [lazydocs](https://github.com/ml-tooling/lazydocs)._ diff --git a/docs/docstring/utils.md b/docs/docstring/utils.md new file mode 100644 index 0000000..926d36c --- /dev/null +++ b/docs/docstring/utils.md @@ -0,0 +1,117 @@ + + + + +# module `utils` +Exceptions and utilities. + + +--- + + + +## function `camel_to_snake` + +```python +camel_to_snake(name: str) → str +``` + +Converts camelCase to snake_case. + + + +**Args:** + + - `name`: String to convert. + + + +**Returns:** + Converted string. + + +--- + + + +## function `unmarshal` + +```python +unmarshal(j: Any, o: Type, convert_camel_case=True) → Any +``` + +Unmarshal a JSON string to a TES model. + + + +**Args:** + + - `j`: JSON string or dictionary to unmarshal. + - `o`: TES model to unmarshal to. + - `convert_camel_case`: Convert values in `j` from camelCase to snake_case. + + + +**Returns:** + Unmarshalled TES model. + + + +**Raises:** + + - `UnmarshalError`: If `j` cannot be unmarshalled to `o`. + + +--- + + + +## class `UnmarshalError` +Raised when a JSON string cannot be unmarshalled to a TES model. + + + +### method `__init__` + +```python +__init__(*args, **kwargs) +``` + + + + + + + + + +--- + + + +## class `TimeoutError` + + + + + + +### method `__init__` + +```python +__init__(*args, **kwargs) +``` + + + + + + + + + + + +--- + +_This file was automatically generated via [lazydocs](https://github.com/ml-tooling/lazydocs)._ diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000..1833352 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,5 @@ +lazydocs>=0.4.8 +mkdocs>=1.4.2 +mkdocs-awesome-pages-plugin>=2.8.0 +mkdocs-material>=9.0.12 +pydocstyle>=6.3.0 diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 0000000..906634c --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,15 @@ +site_name: py-tes documentation +site_url: https://ohsu-comp-bio.github.io/py-tes +site_author: Kyle Ellrott + +repo_name: py-tes +repo_url: https://github.com/ohsu-comp-bio/py-tes +edit_uri: edit/master/docs +docs_dir: docs + +theme: + name: material + +plugins: + - awesome-pages + - search diff --git a/requirements.txt b/requirements.txt index 631e096..907fd76 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,3 @@ -attrs>=17.4.0 -future>=0.16.0 -python-dateutil>=2.6.1 -requests>=2.18.2 +attrs>=22.2.0 +python-dateutil>=2.8.2 +requests>=2.28.2 diff --git a/setup.py b/setup.py index b75e855..3b0898b 100644 --- a/setup.py +++ b/setup.py @@ -36,26 +36,21 @@ def find_version(*file_paths): maintainer_email="kellrott@gmail.com", url="https://github.com/ohsu-comp-bio/py-tes", license="MIT", - packages=find_packages(), - python_requires=">=2.7, <4", - install_requires=[ - "attrs>=17.4.0", - "future>=0.16.0", - "python-dateutil>=2.6.1", - "requests>=2.18.1" - ], + packages=find_packages(exclude=["tests*"]), + python_requires=">=3.7, <4", + install_requires=read("requirements.txt").splitlines(), zip_safe=True, classifiers=[ - "Development Status :: 3 - Alpha", + "Development Status :: 4 - Beta", "Intended Audience :: Developers", "Natural Language :: English", "License :: OSI Approved :: MIT License", "Topic :: Software Development :: Libraries", - "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.4", - "Programming Language :: Python :: 3.5", - "Programming Language :: Python :: 3.6" + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9" + "Programming Language :: Python :: 3.10" + "Programming Language :: Python :: 3.11" ], ) diff --git a/tes/__init__.py b/tes/__init__.py index 95b0822..28efe4e 100644 --- a/tes/__init__.py +++ b/tes/__init__.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from tes.client import HTTPClient from tes.utils import unmarshal from tes.models import ( @@ -20,22 +18,22 @@ ) __all__ = [ - HTTPClient, - unmarshal, - Input, - Output, - Resources, - Executor, - Task, - ExecutorLog, - TaskLog, - OutputFileLog, - CreateTaskResponse, - GetTaskRequest, - ListTasksRequest, - ListTasksResponse, - ServiceInfoRequest, - ServiceInfo + "HTTPClient", + "unmarshal", + "Input", + "Output", + "Resources", + "Executor", + "Task", + "ExecutorLog", + "TaskLog", + "OutputFileLog", + "CreateTaskResponse", + "GetTaskRequest", + "ListTasksRequest", + "ListTasksResponse", + "ServiceInfoRequest", + "ServiceInfo" ] -__version__ = "0.4.2" +__version__ = "1.1.0-rc.1" diff --git a/tes/client.py b/tes/client.py index 6c2fdf7..01d2f25 100644 --- a/tes/client.py +++ b/tes/client.py @@ -1,4 +1,4 @@ -from __future__ import absolute_import, print_function, unicode_literals +"""TES access methods and helper functions.""" import re import requests @@ -6,8 +6,8 @@ from attr import attrs, attrib from attr.validators import instance_of, optional -from builtins import str -from requests.utils import urlparse +from urllib.parse import urlparse +from typing import Any, Dict, List, Optional from tes.models import (Task, ListTasksRequest, ListTasksResponse, ServiceInfo, GetTaskRequest, CancelTaskRequest, CreateTaskResponse, @@ -15,26 +15,119 @@ from tes.utils import unmarshal, TimeoutError +def append_suffixes_to_url( + urls: List[str], suffixes: List[str] +) -> List[str]: + """Compile all combinations of full paths from paths and suffixes. + + Args: + urls: List of URL paths. + prefixes: List of suffixes to be appended to `urls`. + + Returns: + List of full path combinations, in the provided order of `paths` and + `suffixes`, starting with all suffix combinations for the first path, + then those for the second path, and so on. Paths are stripped of any + trailing slashes. + + Examples: + >>> client = tes.HTTPClient.append_suffixes_to_url(['https://funnel.exa + mple.com'], ['ga4gh/tes/v1', 'v1', '']) + ['https://funnel.example.com/ga4gh/tes/v1', 'https://funnel.example.com + /v1', 'https://funnel.example.com'] + """ + compiled_paths: List[str] = [] + for url in urls: + for suffix in suffixes: + compiled_paths.append( + f"{url.rstrip('/')}/{suffix.strip('/')}".rstrip('/')) + return compiled_paths + + +def send_request( + paths: List[str], method: str = 'get', + kwargs_requests: Optional[Dict[str, Any]] = None, **kwargs: Any +) -> requests.Response: + """Send request to a list of URLs, returning the first valid response. + + Args: + paths: List of fully qualified URLs. + method: HTTP method to use for the request; one of 'get' (default), + 'post', 'put', and 'delete'. + kwargs_requests: Keyword arguments to pass to the :mod:`requests` call. + **kwargs: Keyword arguments for path parameter substition. + + Returns: + The first successful response from the list of endpoints. + + Raises: + requests.exceptions.HTTPError: If no response is received from any + path. + requests.exceptions.HTTPError: As soon as the first 4xx or 5xx status + code is received. + requests.exceptions.HTTPError: If, after trying all paths, at least one + 404 status code and no other 4xx or 5xx status codes are received. + ValueError: If an unsupported HTTP method is provided. + """ + if kwargs_requests is None: + kwargs_requests = {} + if method not in ('get', 'post', 'put', 'delete'): + raise ValueError(f"Unsupported HTTP method: {method}") + + response: requests.Response = requests.Response() + http_exceptions: Dict[str, Exception] = {} + for path in paths: + try: + response = getattr(requests, method)( + path.format(**kwargs), **kwargs_requests) + except requests.exceptions.RequestException as exc: + http_exceptions[path] = exc + continue + if response.status_code != 404: + break + + if response.status_code is None: + raise requests.exceptions.HTTPError( + f"No response received; HTTP Exceptions: {http_exceptions}") + response.raise_for_status() + return response + + def process_url(value): return re.sub("[/]+$", "", value) @attrs class HTTPClient(object): - url = attrib(converter=process_url) - timeout = attrib(default=10, validator=instance_of(int)) - user = attrib(default=None, - converter=strconv, - validator=optional(instance_of(str))) - password = attrib(default=None, - converter=strconv, - validator=optional(instance_of(str))) - token = attrib(default=None, - converter=strconv, - validator=optional(instance_of(str))) - - @url.validator + """HTTP client class for interacting with the TES API.""" + url: str = attrib(converter=process_url, validator=instance_of(str)) + timeout: int = attrib(default=10, validator=instance_of(int)) + user: Optional[str] = attrib( + default=None, converter=strconv, validator=optional(instance_of(str))) + password: Optional[str] = attrib( + default=None, converter=strconv, validator=optional(instance_of(str))) + token: Optional[str] = attrib( + default=None, converter=strconv, validator=optional(instance_of(str))) + + def __attrs_post_init__(self): + # for backward compatibility + self.urls: List[str] = append_suffixes_to_url( + [self.url], ["/ga4gh/tes/v1", "/v1", "/"] + ) + + @url.validator # type: ignore def __check_url(self, attribute, value): + """Validate URL scheme of TES instance. + + `attrs` validator function for `HTTPClient.url`. + + Args: + attribute: Attribute being validated. + value: Attribute value. + + Raises: + ValueError: If URL scheme is unsupported. + """ u = urlparse(value) if u.scheme not in ["http", "https"]: raise ValueError( @@ -42,48 +135,87 @@ def __check_url(self, attribute, value): % ("http", "https") ) - def get_service_info(self): - kwargs = self._request_params() - response = requests.get( - "%s/v1/tasks/service-info" % (self.url), - **kwargs) - response.raise_for_status() + def get_service_info(self) -> ServiceInfo: + """Access method for `GET /service-info`. + + Returns: + `tes.models.ServiceInfo` instance. + """ + kwargs: Dict[str, Any] = self._request_params() + paths = append_suffixes_to_url( + self.urls, ["service-info", "tasks/service-info"] + ) + response = send_request(paths=paths, kwargs_requests=kwargs) return unmarshal(response.json(), ServiceInfo) - def create_task(self, task): + def create_task(self, task: Task) -> CreateTaskResponse: + """Access method for `POST /tasks`. + + Args: + task: `tes.models.Task` instance. + + Returns: + `tes.models.CreateTaskResponse` instance. + + Raises: + TypeError: If `task` is not a `tes.models.Task` instance. + """ if isinstance(task, Task): msg = task.as_json() else: raise TypeError("Expected Task instance") - kwargs = self._request_params(data=msg) - response = requests.post( - "%s/v1/tasks" % (self.url), - **kwargs - ) - response.raise_for_status() + kwargs: Dict[str, Any] = self._request_params(data=msg) + paths = append_suffixes_to_url(self.urls, ["/tasks"]) + response = send_request(paths=paths, method='post', + kwargs_requests=kwargs) return unmarshal(response.json(), CreateTaskResponse).id - def get_task(self, task_id, view="BASIC"): - req = GetTaskRequest(task_id, view) - payload = {"view": req.view} - kwargs = self._request_params(params=payload) - response = requests.get( - "%s/v1/tasks/%s" % (self.url, req.id), - **kwargs) - response.raise_for_status() + def get_task(self, task_id: str, view: str = "BASIC") -> Task: + """Access method for `GET /tasks/{id}`. + + Args: + task_id: TES Task ID. + view: Task info verbosity. One of `MINIMAL`, `BASIC` and `FULL`. + + Returns: + `tes.models.Task` instance. + """ + req: GetTaskRequest = GetTaskRequest(task_id, view) + payload: Dict[str, Optional[str]] = {"view": req.view} + kwargs: Dict[str, Any] = self._request_params(params=payload) + paths = append_suffixes_to_url(self.urls, ["/tasks/{task_id}"]) + response = send_request(paths=paths, kwargs_requests=kwargs, + task_id=req.id) return unmarshal(response.json(), Task) - def cancel_task(self, task_id): - req = CancelTaskRequest(task_id) - kwargs = self._request_params() - response = requests.post( - "%s/v1/tasks/%s:cancel" % (self.url, req.id), - **kwargs) - response.raise_for_status() - return + def cancel_task(self, task_id: str) -> None: + """Access method for `POST /tasks/{id}:cancel`. - def list_tasks(self, view="MINIMAL", page_size=None, page_token=None): + Args: + task_id: TES Task ID. + """ + req: CancelTaskRequest = CancelTaskRequest(task_id) + kwargs: Dict[str, Any] = self._request_params() + paths = append_suffixes_to_url(self.urls, ["/tasks/{task_id}:cancel"]) + send_request(paths=paths, method='post', kwargs_requests=kwargs, + task_id=req.id) + return None + + def list_tasks( + self, view: str = "MINIMAL", page_size: Optional[int] = None, + page_token: Optional[str] = None + ) -> ListTasksResponse: + """Access method for `GET /tasks`. + + Args: + view: Task info verbosity. One of `MINIMAL`, `BASIC` and `FULL`. + page_size: Number of tasks to return. + page_token: Token to retrieve the next page of tasks. + + Returns: + `tes.models.ListTasksResponse` instance. + """ req = ListTasksRequest( view=view, page_size=page_size, @@ -91,47 +223,55 @@ def list_tasks(self, view="MINIMAL", page_size=None, page_token=None): name_prefix=None, project=None ) - msg = req.as_dict() + msg: Dict = req.as_dict() - kwargs = self._request_params(params=msg) - response = requests.get( - "%s/v1/tasks" % (self.url), - **kwargs) - response.raise_for_status() + kwargs: Dict[str, Any] = self._request_params(params=msg) + paths = append_suffixes_to_url(self.urls, ["/tasks"]) + response = send_request(paths=paths, kwargs_requests=kwargs) return unmarshal(response.json(), ListTasksResponse) - def wait(self, task_id, timeout=None): - def check_success(data): + def wait(self, task_id: str, timeout=None) -> Task: + def check_success(data: Task) -> bool: return data.state not in ["QUEUED", "RUNNING", "INITIALIZING"] max_time = time.time() + timeout if timeout else None + response: Optional[Task] = None while True: try: response = self.get_task(task_id, "MINIMAL") except Exception: - response = None + raise Exception(f"Failed to get task {task_id}") if response is not None: if check_success(response): return response - if max_time is not None and time.time() >= max_time: - raise TimeoutError("last_response: %s" % (response.as_dict())) + if max_time is not None and time.time() >= max_time: + raise TimeoutError("last_response: {response.as_dict()}") time.sleep(0.5) - def _request_params(self, data=None, params=None): - kwargs = {'timeout': self.timeout} + def _request_params( + self, data: Optional[str] = None, params: Optional[Dict] = None + ) -> Dict[str, Any]: + """Compile request parameters. + + Args: + data: JSON payload to be sent in the request body. + Returns: + Dictionary of request parameters. + """ + kwargs: Dict[str, Any] = {} + kwargs['timeout'] = self.timeout + kwargs['headers'] = {} + kwargs['headers']['Content-type'] = 'application/json' + if self.user is not None and self.password is not None: + kwargs['auth'] = (self.user, self.password) if data: kwargs['data'] = data if params: kwargs['params'] = params if self.token: - kwargs['headers'] = {'Content-type': 'application/json', - 'Authorization': 'Bearer ' + self.token} - else: - kwargs['headers'] = {'Content-type': 'application/json'} - kwargs['auth'] = (self.user, self.password) - + kwargs['headers']['Authorization'] = f"Bearer {self.token}" return kwargs diff --git a/tes/models.py b/tes/models.py index f0f50c2..e5cdf6e 100644 --- a/tes/models.py +++ b/tes/models.py @@ -1,47 +1,56 @@ +"""TES models, converters, validators and helpers.""" + from __future__ import absolute_import, print_function, unicode_literals import dateutil.parser import json import os -import six from attr import asdict, attrs, attrib from attr.validators import instance_of, optional, in_ -from builtins import str from datetime import datetime +from typing import Any, Dict, List, Optional, Tuple, Type, Union -@attrs +@attrs(repr=False) class _ListOfValidator(object): - type = attrib() + """`attrs` validator class for lists.""" - def __call__(self, inst, attr, value): - """ - We use a callable class to be able to change the ``__repr__``. - """ + type: Type = attrib() + + def __call__(self, inst, attr, value) -> None: + """We use a callable class to be able to change the ``__repr__``.""" if not all([isinstance(n, self.type) for n in value]): raise TypeError( - "'{name}' must be a list of {type!r} (got {value!r} that is a " - "list of {actual!r})." - .format(name=attr.name, - type=self.type, - actual=value[0].__class__, - value=value), - attr, self.type, value, + f"'{attr.name}' must be a list of {self.type!r} (got " + f"{value!r}", attr ) - def __repr__(self): - return ( - "" - .format(type=self.type) - ) + def __repr__(self) -> str: + return f"" + + +def list_of(_type: Any) -> _ListOfValidator: + """`attrs` validator for lists of a given type. + + Args: + _type: Type to validate. + + Returns: + `attrs` validator for the given type. + """ + return _ListOfValidator(_type) -def list_of(type): - return _ListOfValidator(type) +def _drop_none(obj: Any) -> Any: + """Drop `None` values from a nested data structure. + Args: + obj: Object to process. -def _drop_none(obj): + Returns: + Object with `None` values removed. + """ if isinstance(obj, (list, tuple, set)): return type(obj)(_drop_none(x) for x in obj if x is not None) elif isinstance(obj, dict): @@ -53,13 +62,22 @@ def _drop_none(obj): return obj -def strconv(value): +def strconv(value: Any) -> Any: + """Explicitly cast a string-like value or list thereof to string(s). + + Args: + value: Value to convert. + + Returns: + Converted value. If `value` is a list, all elements are converted to + strings. If `value` is not string-like, it will be returned as is. + """ if isinstance(value, (tuple, list)): - if all([isinstance(n, six.string_types) for n in value]): + if all([isinstance(n, str) for n in value]): return [str(n) for n in value] else: return value - elif isinstance(value, six.string_types): + elif isinstance(value, str): return str(value) else: return value @@ -67,19 +85,46 @@ def strconv(value): # since an int64 value is encoded as a string in json we need to handle # conversion -def int64conv(value): +def int64conv(value: Optional[str]) -> Optional[int]: + """Convert string to `int64`. + + Args: + value: String to convert. + + Returns: + Converted value. + """ if value is not None: return int(value) return value -def timestampconv(value): - if isinstance(value, six.string_types): - return dateutil.parser.parse(value) - return value +def timestampconv(value: Optional[str]) -> Optional[datetime]: + """Convert string to `datetime`. + + Args: + value: String to convert. + + Returns: + Converted value. + """ + if value is None: + return value + return dateutil.parser.parse(value) + + +def datetime_json_handler(x: Any) -> str: + """JSON handler for `datetime` objects. + Args: + x: Object to convert. -def datetime_json_handler(x): + Returns: + Converted object. + + Raises: + TypeError: If `x` is not a `datetime` object. + """ if isinstance(x, datetime): return x.isoformat() raise TypeError("Unknown type") @@ -87,213 +132,250 @@ def datetime_json_handler(x): @attrs class Base(object): + """`attrs` base class for all TES and helper models.""" - def as_dict(self, drop_empty=True): + def as_dict(self, drop_empty: bool = True) -> Dict[str, Any]: obj = asdict(self) if drop_empty: return _drop_none(obj) return obj - def as_json(self, drop_empty=True): + def as_json(self, drop_empty: bool = True, **kwargs) -> str: return json.dumps( self.as_dict(drop_empty), - default=datetime_json_handler + default=datetime_json_handler, + **kwargs ) @attrs class Input(Base): - url = attrib( + """TES `tesInput` `attrs` model class.""" + + url: Optional[str] = attrib( default=None, converter=strconv, validator=optional(instance_of(str)) ) - path = attrib( + path: Optional[str] = attrib( default=None, converter=strconv, validator=optional(instance_of(str)) ) - type = attrib( + type: str = attrib( default="FILE", validator=in_(["FILE", "DIRECTORY"]) ) - name = attrib( + name: Optional[str] = attrib( default=None, converter=strconv, validator=optional(instance_of(str)) ) - description = attrib( + description: Optional[str] = attrib( default=None, converter=strconv, validator=optional(instance_of(str)) ) - content = attrib( + content: Optional[str] = attrib( default=None, converter=strconv, validator=optional(instance_of(str)) ) + streamable: Optional[bool] = attrib( + default=None, validator=optional(instance_of(bool)) + ) @attrs class Output(Base): - url = attrib( + """TES `tesOutput` `attrs` model class.""" + + url: Optional[str] = attrib( default=None, converter=strconv, validator=optional(instance_of(str)) ) - path = attrib( + path: Optional[str] = attrib( default=None, converter=strconv, validator=optional(instance_of(str)) ) - type = attrib( + type: str = attrib( default="FILE", validator=in_(["FILE", "DIRECTORY"]) ) - name = attrib( + name: Optional[str] = attrib( default=None, converter=strconv, validator=optional(instance_of(str)) ) - description = attrib( + description: Optional[str] = attrib( default=None, converter=strconv, validator=optional(instance_of(str)) ) @attrs class Resources(Base): - cpu_cores = attrib( + """TES `tesResources` `attrs` model class.""" + + cpu_cores: Optional[int] = attrib( default=None, validator=optional(instance_of(int)) ) - ram_gb = attrib( + ram_gb: Optional[Union[float, int]] = attrib( default=None, validator=optional(instance_of((float, int))) ) - disk_gb = attrib( + disk_gb: Optional[Union[float, int]] = attrib( default=None, validator=optional(instance_of((float, int))) ) - preemptible = attrib( + preemptible: Optional[bool] = attrib( default=None, validator=optional(instance_of(bool)) ) - zones = attrib( + zones: Optional[List[str]] = attrib( default=None, converter=strconv, validator=optional(list_of(str)) ) + backend_parameters: Optional[List[str]] = attrib( + default=None, converter=strconv, validator=optional(instance_of(list)) + ) + backend_parameters_strict: Optional[bool] = attrib( + default=None, validator=optional(instance_of(bool)) + ) @attrs class Executor(Base): - image = attrib( + """TES `tesExecutor` `attrs` model class.""" + + image: str = attrib( converter=strconv, validator=instance_of(str) ) - command = attrib( + command: List[str] = attrib( converter=strconv, validator=list_of(str) ) - workdir = attrib( + ignore_error: str = attrib( + default=None, converter=strconv, validator=optional(instance_of(bool)) + ) + workdir: str = attrib( default=None, converter=strconv, validator=optional(instance_of(str)) ) - stdin = attrib( + stdin: str = attrib( default=None, converter=strconv, validator=optional(instance_of(str)) ) - stdout = attrib( + stdout: str = attrib( default=None, converter=strconv, validator=optional(instance_of(str)) ) - stderr = attrib( + stderr: str = attrib( default=None, converter=strconv, validator=optional(instance_of(str)) ) - env = attrib( + env: Optional[Dict] = attrib( default=None, validator=optional(instance_of(dict)) ) @attrs class ExecutorLog(Base): - start_time = attrib( + """TES `tesExecutorLog` `attrs` model class.""" + + start_time: datetime = attrib( default=None, converter=timestampconv, validator=optional(instance_of(datetime)) ) - end_time = attrib( + end_time: datetime = attrib( default=None, converter=timestampconv, validator=optional(instance_of(datetime)) ) - stdout = attrib( + stdout: Optional[str] = attrib( default=None, converter=strconv, validator=optional(instance_of(str)) ) - stderr = attrib( + stderr: Optional[str] = attrib( default=None, converter=strconv, validator=optional(instance_of(str)) ) - exit_code = attrib( + exit_code: Optional[int] = attrib( default=None, validator=optional(instance_of(int)) ) @attrs class OutputFileLog(Base): - url = attrib( + """TES `tesOutputFileLog` `attrs` model class.""" + + url: Optional[str] = attrib( default=None, converter=strconv, validator=optional(instance_of(str)) ) - path = attrib( + path: Optional[str] = attrib( default=None, converter=strconv, validator=optional(instance_of(str)) ) - size_bytes = attrib( + size_bytes: Optional[int] = attrib( default=None, converter=int64conv, validator=optional(instance_of(int)) ) @attrs class TaskLog(Base): - start_time = attrib( + """TES `tesTaskLog` `attrs` model class.""" + + start_time: datetime = attrib( default=None, converter=timestampconv, validator=optional(instance_of(datetime)) ) - end_time = attrib( + end_time: datetime = attrib( default=None, converter=timestampconv, validator=optional(instance_of(datetime)) ) - metadata = attrib( + metadata: Optional[Dict] = attrib( default=None, validator=optional(instance_of(dict)) ) - logs = attrib( + logs: Optional[List[ExecutorLog]] = attrib( default=None, validator=optional(list_of(ExecutorLog)) ) - outputs = attrib( + outputs: Optional[List[OutputFileLog]] = attrib( default=None, validator=optional(list_of(OutputFileLog)) ) - system_logs = attrib( + system_logs: Optional[List[str]] = attrib( default=None, validator=optional(list_of(str)) ) @attrs class Task(Base): - id = attrib( + """TES `tesTask` `attrs` model class.""" + + id: Optional[str] = attrib( default=None, converter=strconv, validator=optional(instance_of(str)) ) - state = attrib( + state: Optional[str] = attrib( default=None, validator=optional(in_( ["UNKNOWN", "QUEUED", "INITIALIZING", "RUNNING", "COMPLETE", "CANCELED", "EXECUTOR_ERROR", "SYSTEM_ERROR"] )) ) - name = attrib( + name: Optional[str] = attrib( default=None, converter=strconv, validator=optional(instance_of(str)) ) - description = attrib( + description: Optional[str] = attrib( default=None, converter=strconv, validator=optional(instance_of(str)) ) - inputs = attrib( + inputs: Optional[List[Input]] = attrib( default=None, validator=optional(list_of(Input)) ) - outputs = attrib( + outputs: Optional[List[Output]] = attrib( default=None, validator=optional(list_of(Output)) ) - resources = attrib( + resources: Optional[Resources] = attrib( default=None, validator=optional(instance_of(Resources)) ) - executors = attrib( + executors: Optional[List[Executor]] = attrib( default=None, validator=optional(list_of(Executor)) ) - volumes = attrib( + volumes: Optional[List[str]] = attrib( default=None, validator=optional(list_of(str)) ) - tags = attrib( + tags: Optional[Dict] = attrib( default=None, validator=optional(instance_of(dict)) ) - logs = attrib( + logs: Optional[List[TaskLog]] = attrib( default=None, validator=optional(list_of(TaskLog)) ) - creation_time = attrib( + creation_time: datetime = attrib( default=None, converter=timestampconv, validator=optional(instance_of(datetime)) ) - def is_valid(self): + def is_valid(self) -> Tuple[bool, Union[None, TypeError]]: + """Validate a `Task` model instance. + + Returns: + A tuple containing a boolean indicating whether the model is + valid, and a `TypeError` if the model is invalid, or `None` if it + is. + """ errs = [] if self.executors is None or len(self.executors) == 0: errs.append("Must provide one or more Executors") @@ -301,7 +383,7 @@ def is_valid(self): for e in self.executors: if e.image is None: errs.append("Executor image must be provided") - if len(e.command) == 0: + if e.command is None or len(e.command) == 0: errs.append("Executor command must be provided") if e.stdin is not None: if not os.path.isabs(e.stdin): @@ -313,8 +395,8 @@ def is_valid(self): if not os.path.isabs(e.stderr): errs.append("Executor stderr must be an absolute path") if e.env is not None: - for k, v in self.executors.env: - if not isinstance(k, str) and not isinstance(k, str): + for k, v in e.env.items(): + if not isinstance(k, str) and not isinstance(v, str): errs.append( "Executor env keys and values must be StrType" ) @@ -336,7 +418,7 @@ def is_valid(self): errs.append("Output url must be provided") if o.path is None: errs.append("Output path must be provided") - elif not os.path.isabs(i.path): + elif not os.path.isabs(o.path): errs.append("Output path must be absolute") if self.volumes is not None: @@ -346,7 +428,7 @@ def is_valid(self): errs.append("Volume paths must be absolute") if self.tags is not None: - for k, v in self.tags: + for k, v in self.tags.items(): if not isinstance(k, str) and not isinstance(k, str): errs.append( "Tag keys and values must be StrType" @@ -360,75 +442,139 @@ def is_valid(self): @attrs class GetTaskRequest(Base): - id = attrib( + """`attrs` model class for `GET /tasks/{id}` request parameters.""" + + id: str = attrib( converter=strconv, validator=instance_of(str) ) - view = attrib( + view: Optional[str] = attrib( default=None, validator=optional(in_(["MINIMAL", "BASIC", "FULL"])) ) @attrs class CreateTaskResponse(Base): - id = attrib( + """TES `tesCreateTaskResponse` `attrs` model class.""" + + id: str = attrib( converter=strconv, validator=instance_of(str) ) @attrs class ServiceInfoRequest(Base): - pass + """`attrs` model class for `GET /service-info` request parameters.""" + + +@attrs +class Organization: + name: Optional[str] = attrib( + default=None, converter=strconv, validator=optional(instance_of(str)) + ) + url: Optional[str] = attrib( + default=None, converter=strconv, validator=optional(instance_of(str)) + ) + + +@attrs +class Type: + artifact: Optional[str] = attrib( + default=None, converter=strconv, validator=optional(instance_of(str)) + ) + group: Optional[str] = attrib( + default=None, converter=strconv, validator=optional(instance_of(str)) + ) + version: Optional[str] = attrib( + default=None, converter=strconv, validator=optional(instance_of(str)) + ) @attrs class ServiceInfo(Base): - name = attrib( + """TES `tesServiceInfo` `attrs` model class.""" + contact_url: Optional[str] = attrib( default=None, converter=strconv, validator=optional(instance_of(str)) ) - doc = attrib( + created_at: Optional[str] = attrib( default=None, converter=strconv, validator=optional(instance_of(str)) ) - storage = attrib( - default=None, converter=strconv, validator=optional(list_of(str)) + description: Optional[str] = attrib( + default=None, converter=strconv, validator=optional(instance_of(str)) + ) + documentation_url: Optional[str] = attrib( + default=None, converter=strconv, validator=optional(instance_of(str)) + ) + environment: Optional[str] = attrib( + default=None, converter=strconv, validator=optional(instance_of(str)) + ) + id: Optional[str] = attrib( + default=None, converter=strconv, validator=optional(instance_of(str)) + ) + name: Optional[str] = attrib( + default=None, converter=strconv, validator=optional(instance_of(str)) + ) + organization: Optional[dict] = attrib( + default=None, validator=optional(instance_of(dict)) + ) + storage: Optional[List[str]] = attrib( + default=None, converter=strconv, validator=optional(instance_of(list)) + ) + tes_resources_backend_parameters: Optional[List[str]] = attrib( + default=None, converter=strconv, validator=optional(instance_of(list)) + ) + type: Optional[dict] = attrib( + default=None, validator=optional(instance_of(dict)) + ) + updated_at: Optional[str] = attrib( + default=None, converter=strconv, validator=optional(instance_of(str)) + ) + version: Optional[str] = attrib( + default=None, converter=strconv, validator=optional(instance_of(str)) ) @attrs class CancelTaskRequest(Base): - id = attrib( + """`attrs` model class for `POST /tasks/{id}:cancel` request parameters.""" + + id: str = attrib( converter=strconv, validator=instance_of(str) ) @attrs class CancelTaskResponse(Base): - pass + """TES `tesCancelTaskResponse` `attrs` model class.""" @attrs class ListTasksRequest(Base): - project = attrib( + """`attrs` model class for `GET /tasks` request parameters.""" + + project: Optional[str] = attrib( default=None, converter=strconv, validator=optional(instance_of(str)) ) - name_prefix = attrib( + name_prefix: Optional[str] = attrib( default=None, converter=strconv, validator=optional(instance_of(str)) ) - page_size = attrib( + page_size: Optional[int] = attrib( default=None, validator=optional(instance_of(int)) ) - page_token = attrib( + page_token: Optional[str] = attrib( default=None, converter=strconv, validator=optional(instance_of(str)) ) - view = attrib( + view: Optional[str] = attrib( default=None, validator=optional(in_(["MINIMAL", "BASIC", "FULL"])) ) @attrs class ListTasksResponse(Base): - tasks = attrib( + """TES `tesListTasksResponse` `attrs` model class.""" + + tasks: Optional[List[Task]] = attrib( default=None, validator=optional(list_of(Task)) ) - next_page_token = attrib( + next_page_token: Optional[str] = attrib( default=None, converter=strconv, validator=optional(instance_of(str)) ) diff --git a/tes/utils.py b/tes/utils.py index dfec1ac..1f0f4f4 100644 --- a/tes/utils.py +++ b/tes/utils.py @@ -1,8 +1,10 @@ -from __future__ import absolute_import, print_function, unicode_literals +"""Exceptions and utilities.""" import json import re +from typing import Any, Dict, Type + from tes.models import (Task, Input, Output, Resources, Executor, TaskLog, ExecutorLog, OutputFileLog) @@ -11,12 +13,21 @@ all_cap_re = re.compile('([a-z0-9])([A-Z])') -def camel_to_snake(name): +def camel_to_snake(name: str) -> str: + """Converts camelCase to snake_case. + + Args: + name: String to convert. + + Returns: + Converted string. + """ s1 = first_cap_re.sub(r'\1_\2', name) return all_cap_re.sub(r'\1_\2', s1).lower() class UnmarshalError(Exception): + """Raised when a JSON string cannot be unmarshalled to a TES model.""" def __init__(self, *args, **kwargs): Exception.__init__(self, *args, **kwargs) @@ -26,15 +37,36 @@ def __init__(self, *args, **kwargs): Exception.__init__(self, *args, **kwargs) -def unmarshal(j, o, convert_camel_case=True): +def unmarshal(j: Any, o: Type, convert_camel_case=True) -> Any: + """Unmarshal a JSON string to a TES model. + + Args: + j: JSON string or dictionary to unmarshal. + o: TES model to unmarshal to. + convert_camel_case: Convert values in `j` from camelCase to snake_case. + + Returns: + Unmarshalled TES model. + + Raises: + UnmarshalError: If `j` cannot be unmarshalled to `o`. + """ + m: Any = None if isinstance(j, str): - m = json.loads(j) - elif isinstance(j, dict): - m = j + try: + m = json.loads(j) + except json.decoder.JSONDecodeError: + pass + elif j is None: + return None else: - raise TypeError("j must be a str or dict") + m = j + + if not isinstance(m, dict): + raise TypeError("j must be a dictionary, a JSON string evaluation to " + "a dictionary, or None") - d = {} + d: Dict[str, Any] = {} if convert_camel_case: for k, v in m.items(): d[camel_to_snake(k)] = v @@ -61,7 +93,7 @@ def unmarshal(j, o, convert_camel_case=True): } } - def _unmarshal(v, obj): + def _unmarshal(v: Any, obj: Type) -> Any: if isinstance(v, list): field = [] for item in v: @@ -75,16 +107,8 @@ def _unmarshal(v, obj): field = v omap = fullOmap.get(o.__name__, {}) if k in omap: - if isinstance(omap[k], tuple): - try: - obj = omap[k][0] - field = _unmarshal(v, obj) - except Exception: - obj = omap[k][1] - field = _unmarshal(v, obj) - else: - obj = omap[k] - field = _unmarshal(v, obj) + obj = omap[k] + field = _unmarshal(v, obj) r[k] = field try: diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/integration/test_funnel.py b/tests/integration/test_funnel.py new file mode 100644 index 0000000..b56be4c --- /dev/null +++ b/tests/integration/test_funnel.py @@ -0,0 +1,35 @@ +import unittest +import tes + + +class TestTESClient(unittest.TestCase): + def setUp(self): + self.cli = tes.HTTPClient("http://localhost:8000", timeout=5) + self.task = tes.Task( + executors=[ + tes.Executor( + image="alpine", + command=["echo", "hello"] + ) + ] + ) + + def test_task_creation(self): + # Test service info retrieval + service_info = self.cli.get_service_info() + self.assertIsNotNone(service_info) + + # Test task creation + task_id = self.cli.create_task(self.task) + self.assertIsNotNone(task_id) + + # Wait for task to complete + _ = self.cli.wait(task_id) + + # Test task info retrieval + task_info = self.cli.get_task(task_id, view="BASIC") + self.assertIsNotNone(task_info) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/requirements.txt b/tests/requirements.txt index b361eb6..8f67ab0 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,5 +1,6 @@ -coverage==4.5.4 -coveralls>=1.1 -flake8>=3.3.0 -nose>=1.3.7 -requests_mock>=1.3.0 +coverage>=6.5.0 +coveralls>=3.3.1 +flake8>=5.0.4 +pytest>=7.2.1 +pytest-cov>=4.0.0 +requests_mock>=1.10.0 diff --git a/tests/test_client.py b/tests/test_client.py index 6333013..45cd9dc 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -1,9 +1,10 @@ +import pytest import requests import requests_mock import unittest import uuid -from tes.client import HTTPClient +from tes.client import append_suffixes_to_url, HTTPClient, send_request from tes.models import Task, Executor from tes.utils import TimeoutError @@ -24,36 +25,50 @@ class TestHTTPClient(unittest.TestCase): def test_cli(self): cli = HTTPClient(url="http://fakehost:8000//", timeout=5) self.assertEqual(cli.url, "http://fakehost:8000") + self.assertEqual(cli.urls, [ + "http://fakehost:8000/ga4gh/tes/v1", + "http://fakehost:8000/v1", + "http://fakehost:8000"] + ) self.assertEqual(cli.timeout, 5) + with self.assertRaises(TypeError): + cli = HTTPClient(url=8000, timeout=5) # type: ignore + + with self.assertRaises(TypeError): + HTTPClient(url="http://fakehost:8000", timeout="5") # type: ignore + with self.assertRaises(ValueError): HTTPClient(url="fakehost:8000", timeout=5) with self.assertRaises(ValueError): - HTTPClient(url="htpp://fakehost:8000", timeout="5") + HTTPClient(url="htpp://fakehost:8000", timeout=5) # type: ignore def test_create_task(self): with requests_mock.Mocker() as m: m.post( - "%s/v1/tasks" % (self.mock_url), + "%s/ga4gh/tes/v1/tasks" % (self.mock_url), status_code=200, json={"id": self.mock_id} ) self.cli.create_task(self.task) self.assertEqual(m.last_request.text, self.task.as_json()) - self.assertEqual(m.last_request.timeout, self.cli.timeout) + self.assertAlmostEqual(m.last_request.timeout, self.cli.timeout) m.post( - "%s/v1/tasks" % (self.mock_url), + "%s/ga4gh/tes/v1/tasks" % (self.mock_url), status_code=500 ) with self.assertRaises(requests.HTTPError): self.cli.create_task(self.task) + with self.assertRaises(TypeError): + self.cli.create_task('not_a_task_object') # type: ignore + def test_get_task(self): with requests_mock.Mocker() as m: m.get( - "%s/v1/tasks/%s" % (self.mock_url, self.mock_id), + "%s/ga4gh/tes/v1/tasks/%s" % (self.mock_url, self.mock_id), status_code=200, json={ "id": self.mock_id, @@ -63,12 +78,14 @@ def test_get_task(self): self.cli.get_task(self.mock_id, "MINIMAL") self.assertEqual( m.last_request.url, - "%s/v1/tasks/%s?view=MINIMAL" % (self.mock_url, self.mock_id) + "%s/ga4gh/tes/v1/tasks/%s?view=MINIMAL" % ( + self.mock_url, self.mock_id + ) ) - self.assertEqual(m.last_request.timeout, self.cli.timeout) + self.assertAlmostEqual(m.last_request.timeout, self.cli.timeout) m.get( - "%s/v1/tasks/%s" % (self.mock_url, self.mock_id), + requests_mock.ANY, status_code=404 ) with self.assertRaises(requests.HTTPError): @@ -77,7 +94,7 @@ def test_get_task(self): def test_list_tasks(self): with requests_mock.Mocker() as m: m.get( - "%s/v1/tasks" % (self.mock_url), + "%s/ga4gh/tes/v1/tasks" % (self.mock_url), status_code=200, json={ "tasks": [] @@ -86,24 +103,24 @@ def test_list_tasks(self): self.cli.list_tasks() self.assertEqual( m.last_request.url, - "%s/v1/tasks?view=MINIMAL" % (self.mock_url) + "%s/ga4gh/tes/v1/tasks?view=MINIMAL" % (self.mock_url) ) - self.assertEqual(m.last_request.timeout, self.cli.timeout) + self.assertAlmostEqual(m.last_request.timeout, self.cli.timeout) # empty response m.get( - "%s/v1/tasks" % (self.mock_url), + "%s/ga4gh/tes/v1/tasks" % (self.mock_url), status_code=200, json={} ) self.cli.list_tasks() self.assertEqual( m.last_request.url, - "%s/v1/tasks?view=MINIMAL" % (self.mock_url) + "%s/ga4gh/tes/v1/tasks?view=MINIMAL" % (self.mock_url) ) m.get( - "%s/v1/tasks" % (self.mock_url), + "%s/ga4gh/tes/v1/tasks" % (self.mock_url), status_code=500 ) with self.assertRaises(requests.HTTPError): @@ -112,40 +129,51 @@ def test_list_tasks(self): def test_cancel_task(self): with requests_mock.Mocker() as m: m.post( - "%s/v1/tasks/%s:cancel" % (self.mock_url, self.mock_id), + "%s/ga4gh/tes/v1/tasks/%s:cancel" % ( + self.mock_url, self.mock_id), status_code=200, json={} ) self.cli.cancel_task(self.mock_id) self.assertEqual( m.last_request.url, - "%s/v1/tasks/%s:cancel" % (self.mock_url, self.mock_id) + "%s/ga4gh/tes/v1/tasks/%s:cancel" % ( + self.mock_url, self.mock_id) ) - self.assertEqual(m.last_request.timeout, self.cli.timeout) + self.assertAlmostEqual(m.last_request.timeout, self.cli.timeout) m.post( - "%s/v1/tasks/%s:cancel" % (self.mock_url, self.mock_id), + "%s/ga4gh/tes/v1/tasks/%s:cancel" % ( + self.mock_url, self.mock_id), status_code=500 ) with self.assertRaises(requests.HTTPError): self.cli.cancel_task(self.mock_id) + m.post( + requests_mock.ANY, + status_code=404, + json={} + ) + with self.assertRaises(requests.HTTPError): + self.cli.cancel_task(self.mock_id) + def test_get_service_info(self): with requests_mock.Mocker() as m: m.get( - "%s/v1/tasks/service-info" % (self.mock_url), + "%s/ga4gh/tes/v1/service-info" % (self.mock_url), status_code=200, json={} ) self.cli.get_service_info() self.assertEqual( m.last_request.url, - "%s/v1/tasks/service-info" % (self.mock_url) + "%s/ga4gh/tes/v1/service-info" % (self.mock_url) ) - self.assertEqual(m.last_request.timeout, self.cli.timeout) + self.assertAlmostEqual(m.last_request.timeout, self.cli.timeout) m.get( - "%s/v1/tasks/service-info" % (self.mock_url), + "%s/ga4gh/tes/v1/service-info" % (self.mock_url), status_code=500 ) with self.assertRaises(requests.HTTPError): @@ -155,7 +183,7 @@ def test_wait(self): with self.assertRaises(TimeoutError): with requests_mock.Mocker() as m: m.get( - "%s/v1/tasks/%s" % (self.mock_url, self.mock_id), + "%s/ga4gh/tes/v1/tasks/%s" % (self.mock_url, self.mock_id), status_code=200, json={ "id": self.mock_id, @@ -166,7 +194,7 @@ def test_wait(self): with requests_mock.Mocker() as m: m.get( - "%s/v1/tasks/%s" % (self.mock_url, self.mock_id), + "%s/ga4gh/tes/v1/tasks/%s" % (self.mock_url, self.mock_id), [ {"status_code": 200, "json": {"id": self.mock_id, "state": "INITIALIZING"}}, @@ -177,3 +205,150 @@ def test_wait(self): ] ) self.cli.wait(self.mock_id, timeout=2) + + def test_wait_exception(self): + with requests_mock.Mocker() as m: + m.get( + "%s/ga4gh/tes/v1/tasks/%s" % (self.mock_url, self.mock_id), + status_code=200, + json={ + "Error": "Error", + } + ) + with self.assertRaises(Exception): + self.cli.wait(self.mock_id, timeout=2) + + def test_wait_no_state_change(self): + with requests_mock.Mocker() as m: + m.get( + "%s/ga4gh/tes/v1/tasks/%s" % (self.mock_url, self.mock_id), + [ + {"status_code": 200, "json": {"id": self.mock_id, "state": "RUNNING"}}, + {"status_code": 200, "json": {"id": self.mock_id, "state": "RUNNING"}}, + # Continues to return RUNNING state + ] + ) + with self.assertRaises(TimeoutError): + self.cli.wait(self.mock_id, timeout=2) + + def test_request_params(self): + + cli = HTTPClient(url="http://fakehost:8000", timeout=5) + vals = cli._request_params() + self.assertAlmostEqual(vals["timeout"], 5) + self.assertEqual(vals["headers"]["Content-type"], "application/json") + self.assertRaises(KeyError, lambda: vals["headers"]["Authorization"]) + self.assertRaises(KeyError, lambda: vals["auth"]) + self.assertRaises(KeyError, lambda: vals["data"]) + self.assertRaises(KeyError, lambda: vals["params"]) + + cli = HTTPClient(url="http://fakehost:8000", user="user", + password="password", token="token") + vals = cli._request_params(data='{"json": "string"}', + params={"query_param": "value"}) + self.assertAlmostEqual(vals["timeout"], 10) + self.assertEqual(vals["headers"]["Content-type"], "application/json") + self.assertEqual(vals["headers"]["Authorization"], "Bearer token") + self.assertEqual(vals["auth"], ("user", "password")) + self.assertEqual(vals["data"], '{"json": "string"}') + self.assertEqual(vals["params"], {"query_param": "value"}) + + def test_append_suffixes_to_url(self): + urls = ["http://example.com", "http://example.com/"] + urls_order = ["http://example1.com", "http://example2.com"] + suffixes = ["foo", "/foo", "foo/", "/foo/"] + no_suffixes = ["", "/", "//", "///"] + suffixes_order = ["1", "2"] + + results = append_suffixes_to_url(urls=urls, suffixes=suffixes) + assert len(results) == len(urls) * len(suffixes) + assert all(url == 'http://example.com/foo' for url in results) + + results = append_suffixes_to_url(urls=urls, suffixes=no_suffixes) + assert len(results) == len(urls) * len(no_suffixes) + assert all(url == 'http://example.com' for url in results) + + results = append_suffixes_to_url(urls=urls_order, suffixes=suffixes_order) + assert len(results) == len(urls_order) * len(suffixes_order) + assert results[0] == 'http://example1.com/1' + assert results[1] == 'http://example1.com/2' + assert results[2] == 'http://example2.com/1' + assert results[3] == 'http://example2.com/2' + + def test_send_request(self): + mock_url = "http://example.com" + mock_id = "mock_id" + mock_urls = append_suffixes_to_url([mock_url], ["/suffix", "/"]) + + # invalid method + with pytest.raises(ValueError): + send_request(paths=mock_urls, method="invalid") + + # errors for all paths + with requests_mock.Mocker() as m: + m.get(requests_mock.ANY, exc=requests.exceptions.ConnectTimeout) + with pytest.raises(requests.HTTPError): + send_request(paths=mock_urls) + + # error on first path, 200 on second + with requests_mock.Mocker() as m: + m.get(mock_urls[0], exc=requests.exceptions.ConnectTimeout) + m.get(mock_urls[1], status_code=200) + response = send_request(paths=mock_urls) + assert response.status_code == 200 + assert m.last_request.url.rstrip('/') == f"{mock_url}" + + # error on first path, 404 on second + with requests_mock.Mocker() as m: + m.get(mock_urls[0], exc=requests.exceptions.ConnectTimeout) + m.get(mock_urls[1], status_code=404) + with pytest.raises(requests.HTTPError): + send_request(paths=mock_urls) + + # 404 on first path, error on second + with requests_mock.Mocker() as m: + m.get(mock_urls[0], status_code=404) + m.get(mock_urls[1], exc=requests.exceptions.ConnectTimeout) + with pytest.raises(requests.HTTPError): + send_request(paths=mock_urls) + + # 404 on first path, 200 on second + with requests_mock.Mocker() as m: + m.get(mock_urls[0], status_code=404) + m.get(mock_urls[1], status_code=200) + response = send_request(paths=mock_urls) + assert response.status_code == 200 + assert m.last_request.url.rstrip('/') == f"{mock_url}" + + # POST 200 + with requests_mock.Mocker() as m: + m.post(f"{mock_url}/suffix/foo/{mock_id}:bar", status_code=200) + paths = append_suffixes_to_url(mock_urls, ["/foo/{id}:bar"]) + response = send_request(paths=paths, method="post", json={}, + id=mock_id) + assert response.status_code == 200 + assert m.last_request.url == f"{mock_url}/suffix/foo/{mock_id}:bar" + + # GET 200 + with requests_mock.Mocker() as m: + m.get(f"{mock_url}/suffix/foo/{mock_id}", status_code=200) + paths = append_suffixes_to_url(mock_urls, ["/foo/{id}"]) + response = send_request(paths=paths, id=mock_id) + assert response.status_code == 200 + assert m.last_request.url == f"{mock_url}/suffix/foo/{mock_id}" + + # POST 404 + with requests_mock.Mocker() as m: + m.post(requests_mock.ANY, status_code=404, json={}) + paths = append_suffixes_to_url(mock_urls, ["/foo"]) + with pytest.raises(requests.HTTPError): + send_request(paths=paths, method="post", json={}) + assert m.last_request.url == f"{mock_url}/foo" + + # GET 500 + with requests_mock.Mocker() as m: + m.get(f"{mock_url}/suffix/foo", status_code=500) + paths = append_suffixes_to_url(mock_urls, ["/foo"]) + with pytest.raises(requests.HTTPError): + send_request(paths=paths) + assert m.last_request.url == f"{mock_url}/suffix/foo" diff --git a/tests/test_models.py b/tests/test_models.py index 2c8dd95..c1cebd7 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -1,67 +1,250 @@ import json import unittest -from tes.models import Task, Executor, Input, Output, strconv +from copy import deepcopy +from tes.models import ( + Executor, + ExecutorLog, + Input, + Output, + OutputFileLog, + Resources, + Task, + TaskLog, + datetime_json_handler, + int64conv, + list_of, + strconv, + timestampconv, + _drop_none, +) -class TestModels(unittest.TestCase): - task = Task( - executors=[ - Executor( - image="alpine", - command=["echo", "hello"] - ) - ] - ) - - expected = { - "executors": [ - { - "image": "alpine", - "command": ["echo", "hello"] - } - ] - } - def test_strconv(self): - self.assertTrue(strconv("foo"), u"foo") - self.assertTrue(strconv(["foo", "bar"]), [u"foo", u"bar"]) - self.assertTrue(strconv(("foo", "bar")), (u"foo", u"bar")) - self.assertTrue(strconv(1), 1) +task_valid = Task( + executors=[ + Executor( + image="alpine", + command=["echo", "hello"] + ) + ] +) + + +datetm = "2018-01-01T00:00:00Z" +task_valid_full = Task( + id="foo", + state="COMPLETE", + name="some_task", + description="some description", + resources=Resources( + cpu_cores=1, + ram_gb=2, + disk_gb=3, + preemptible=True, + zones=["us-east-1", "us-west-1"], + ), + executors=[ + Executor( + image="alpine", + command=["echo", "hello"], + workdir="/abs/path", + stdin="/abs/path", + stdout="/abs/path", + stderr="/abs/path", + env={"VAR": "value"} + ), + Executor( + image="alpine", + command=["echo", "worls"] + ) + ], + inputs=[ + Input( + url="s3:/some/path", + path="/abs/path" + ), + Input( + content="foo", + path="/abs/path" + ) + ], + outputs=[ + Output( + url="s3:/some/path", + path="/abs/path" + ) + ], + volumes=[], + tags={"key": "value", "key2": "value2"}, + logs=[ + TaskLog( + start_time=datetm, # type: ignore + end_time=datetm, # type: ignore + metadata={"key": "value", "key2": "value2"}, + logs=[ + ExecutorLog( + start_time=datetm, # type: ignore + end_time=datetm, # type: ignore + exit_code=0, + stdout="hello", + stderr="world" + ) + ], + outputs=[ + OutputFileLog( + url="s3:/some/path", + path="/abs/path", + size_bytes=int64conv(123) # type: ignore + ) + ], + system_logs=[ + "some system log message", + "some other system log message" + ] + ) + ], + creation_time=datetm # type: ignore +) +task_invalid = Task( + executors=[ + Executor( # type: ignore + image="alpine", + command=["echo", "hello"], + stdin="relative/path", + stdout="relative/path", + stderr="relative/path", + env={1: 2} + ) + ], + inputs=[ + Input( + url="s3:/some/path", + content="foo" + ), + Input( + path="relative/path" + ) + ], + outputs=[ + Output(), + Output( + url="s3:/some/path", + path="relative/path" + ) + ], + volumes=['/abs/path', 'relative/path'], + tags={1: 2} +) + +expected = { + "executors": [ + { + "image": "alpine", + "command": ["echo", "hello"] + } + ] +} + + +class TestModels(unittest.TestCase): + + def test_list_of(self): + validator = list_of(str) + self.assertEqual(list_of(str), validator) + self.assertEqual( + repr(validator), + ">" + ) with self.assertRaises(TypeError): Input( - url="s3:/some/path", path="/opt/foo", content=123 + url="s3:/some/path", + path="/opt/foo", + content=123 # type: ignore ) - - def test_list_of(self): with self.assertRaises(TypeError): Task( inputs=[ Input( url="s3:/some/path", path="/opt/foo" ), - "foo" + "foo" # type: ignore ] ) + def test_drop_none(self): + self.assertEqual(_drop_none({}), {}) + self.assertEqual(_drop_none({"foo": None}), {}) + self.assertEqual(_drop_none({"foo": 1}), {"foo": 1}) + self.assertEqual(_drop_none({"foo": None, "bar": 1}), {"bar": 1}) + self.assertEqual(_drop_none({"foo": [1, None, 2]}), {"foo": [1, 2]}) + self.assertEqual(_drop_none({"foo": {"bar": None}}), {"foo": {}}) + self.assertEqual( + _drop_none({"foo": {"bar": None}, "baz": 1}), + {"foo": {}, "baz": 1} + ) + + def test_strconv(self): + self.assertTrue(strconv("foo"), u"foo") + self.assertTrue(strconv(["foo", "bar"]), [u"foo", u"bar"]) + self.assertTrue(strconv(("foo", "bar")), (u"foo", u"bar")) + self.assertTrue(strconv(1), 1) + self.assertTrue(strconv([1]), [1]) + + def test_int64conv(self): + self.assertEqual(int64conv("1"), 1) + self.assertEqual(int64conv("-1"), -1) + self.assertIsNone(int64conv(None)) + + def test_timestampconv(self): + tm = timestampconv("2018-02-01T00:00:00Z") + self.assertIsNotNone(tm) + assert tm is not None + self.assertAlmostEqual(tm.year, 2018) + self.assertAlmostEqual(tm.month, 2) + self.assertAlmostEqual(tm.day, 1) + self.assertAlmostEqual(tm.hour, 0) + self.assertAlmostEqual(tm.timestamp(), 1517443200.0) + self.assertIsNone(timestampconv(None)) + + def test_datetime_json_handler(self): + tm = timestampconv("2018-02-01T00:00:00Z") + tm_iso = '2018-02-01T00:00:00+00:00' + assert tm is not None + self.assertEqual(datetime_json_handler(tm), tm_iso) + with self.assertRaises(TypeError): + datetime_json_handler(None) + with self.assertRaises(TypeError): + datetime_json_handler("abc") + with self.assertRaises(TypeError): + datetime_json_handler(2001) + with self.assertRaises(TypeError): + datetime_json_handler(tm_iso) + def test_as_dict(self): - self.assertEqual(self.task.as_dict(), self.expected) + task = deepcopy(task_valid) + self.assertEqual(task.as_dict(), expected) + with self.assertRaises(KeyError): + task.as_dict()['inputs'] + self.assertIsNone(task.as_dict(drop_empty=False)['inputs']) def test_as_json(self): - self.assertEqual(self.task.as_json(), json.dumps(self.expected)) + task = deepcopy(task_valid) + self.assertEqual(task.as_json(), json.dumps(expected)) def test_is_valid(self): - self.assertTrue(self.task.is_valid()[0]) + task = deepcopy(task_valid) + self.assertTrue(task.is_valid()[0]) - task2 = self.task - task2.inputs = [Input(path="/opt/foo")] - self.assertFalse(task2.is_valid()[0]) + task = deepcopy(task_valid_full) + self.assertTrue(task.is_valid()[0]) - task3 = self.task - task3.outputs = [ - Output( - url="s3:/some/path", path="foo" - ) - ] - self.assertFalse(task3.is_valid()[0]) + task = deepcopy(task_invalid) + task.executors[0].image = None # type: ignore + task.executors[0].command = None # type: ignore + self.assertFalse(task.is_valid()[0]) + + task = deepcopy(task_invalid) + task.executors = None + self.assertFalse(task.is_valid()[0]) diff --git a/tests/test_utils.py b/tests/test_utils.py index c07dd09..b7fe5ed 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -5,7 +5,23 @@ import unittest from tes.utils import camel_to_snake, unmarshal, UnmarshalError -from tes.models import Input, Task, CreateTaskResponse +from tes.models import ( + CancelTaskRequest, + CancelTaskResponse, + CreateTaskResponse, + Executor, + ExecutorLog, + GetTaskRequest, + Input, + ListTasksRequest, + ListTasksResponse, + Output, + OutputFileLog, + Resources, + ServiceInfo, + Task, + TaskLog, +) class TestUtils(unittest.TestCase): @@ -19,7 +35,110 @@ def test_camel_to_snake(self): self.assertEqual(camel_to_snake(case3), "foo_bar") def test_unmarshal(self): - test_invalid_dict = {"adfasd": "bar"} + + # test unmarshalling with no or minimal contents + try: + unmarshal( + CancelTaskRequest(id="foo").as_json(), + CancelTaskRequest + ) + except Exception: + self.fail("Raised ExceptionType unexpectedly!") + + try: + unmarshal(CancelTaskResponse().as_json(), CancelTaskResponse) + except Exception: + self.fail("Raised ExceptionType unexpectedly!") + + try: + unmarshal( + CreateTaskResponse(id="foo").as_json(), + CreateTaskResponse + ) + except Exception: + self.fail("Raised ExceptionType unexpectedly!") + + try: + unmarshal(Executor( + image="alpine", command=["echo", "hello"]).as_json(), + Executor + ) + except Exception: + self.fail("Raised ExceptionType unexpectedly!") + + try: + unmarshal(ExecutorLog().as_json(), ExecutorLog) + except Exception: + self.fail("Raised ExceptionType unexpectedly!") + + try: + unmarshal( + GetTaskRequest(id="foo", view="BASIC").as_json(), + GetTaskRequest + ) + except Exception: + self.fail("Raised ExceptionType unexpectedly!") + + try: + unmarshal(Input().as_json(), Input) + except Exception: + self.fail("Raised ExceptionType unexpectedly!") + + try: + unmarshal(ListTasksRequest().as_json(), ListTasksRequest) + except Exception: + self.fail("Raised ExceptionType unexpectedly!") + + try: + unmarshal(ListTasksResponse().as_json(), ListTasksResponse) + except Exception: + self.fail("Raised ExceptionType unexpectedly!") + + try: + unmarshal(Output().as_json(), Output) + except Exception: + self.fail("Raised ExceptionType unexpectedly!") + + try: + unmarshal(OutputFileLog().as_json(), OutputFileLog) + except Exception: + self.fail("Raised ExceptionType unexpectedly!") + + try: + unmarshal(Resources().as_json(), Resources) + except Exception: + self.fail("Raised ExceptionType unexpectedly!") + + try: + unmarshal(ServiceInfo().as_json(), ServiceInfo) + except Exception: + self.fail("Raised ExceptionType unexpectedly!") + + try: + unmarshal(Task().as_json(), Task) + except Exception: + self.fail("Raised ExceptionType unexpectedly!") + + try: + unmarshal(TaskLog().as_json(), TaskLog) + except Exception: + self.fail("Raised ExceptionType unexpectedly!") + + # test special cases + self.assertIsNone(unmarshal(None, Input)) + with self.assertRaises(TypeError): + unmarshal([], Input) + with self.assertRaises(TypeError): + unmarshal(1, Input) + with self.assertRaises(TypeError): + unmarshal(1.3, Input) + with self.assertRaises(TypeError): + unmarshal(True, Input) + with self.assertRaises(TypeError): + unmarshal('foo', Input) + + # test with some interesting contents + test_invalid_dict = {"foo": "bar"} test_invalid_str = json.dumps(test_invalid_dict) with self.assertRaises(UnmarshalError): unmarshal(test_invalid_dict, CreateTaskResponse) @@ -33,7 +152,7 @@ def test_unmarshal(self): } test_simple_str = json.dumps(test_simple_dict) o1 = unmarshal(test_simple_dict, Input) - o2 = unmarshal(test_simple_str, Input) + o2 = unmarshal(test_simple_str, Input, convert_camel_case=False) self.assertTrue(isinstance(o1, Input)) self.assertTrue(isinstance(o2, Input)) self.assertEqual(o1, o2) @@ -92,6 +211,13 @@ def test_unmarshal(self): ] } ], + "resources": { + "cpu_cores": 1, + "ram_gb": 2, + "disk_gb": 3, + "preemptible": True, + "zones": ["us-east-1", "us-west-1"] + }, "creation_time": "2017-10-09T17:00:00.0Z" } @@ -100,7 +226,7 @@ def test_unmarshal(self): o2 = unmarshal(test_complex_str, Task) self.assertTrue(isinstance(o1, Task)) self.assertTrue(isinstance(o2, Task)) - self.assertEqual(o1, o2) + self.assertAlmostEqual(o1, o2) expected = test_complex_dict.copy() # handle expected conversions @@ -122,7 +248,6 @@ def test_unmarshal(self): expected["creation_time"] = dateutil.parser.parse( expected["creation_time"] ) - self.assertEqual(o1.as_dict(), expected) def test_unmarshal_types(self):