From 3bfad4f7972b6ae40b88adb4b71b0e48a9c9a993 Mon Sep 17 00:00:00 2001 From: Alex Turbov Date: Mon, 10 Oct 2016 14:52:05 +0700 Subject: [PATCH 01/21] `pathlib` is a part of Python since 3.4 Do not add `pathlib` external dependency for fresh Pythons. --- setup.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4a7d57c0..30c54586 100755 --- a/setup.py +++ b/setup.py @@ -2,6 +2,7 @@ import os import re +import sys try: from setuptools import setup @@ -44,12 +45,14 @@ def rst_strip_code_tag(string): 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3.2', 'Programming Language :: Python :: 3.3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', 'Topic :: Software Development :: Libraries', 'Topic :: System :: Filesystems', ], url='http://github.com/parallels/artifactory', download_url='http://github.com/parallels/artifactory', - install_requires=['pathlib', 'requests', 'python-dateutil'], + install_requires=['requests', 'python-dateutil'] + ['pathlib'] if sys.version_info[0] < 3 or sys.version_info[1] < 4 else [], zip_safe=False, package_data={'': ['README.md']} ) From f6fbb4cc836060f0d23cf11822c449a382d0bb61 Mon Sep 17 00:00:00 2001 From: Alex Turbov Date: Mon, 10 Oct 2016 15:00:20 +0700 Subject: [PATCH 02/21] Fix import errors in tests. Make imports cross-Python, but it doesn't fix the tests set... --- int_test.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/int_test.py b/int_test.py index b26eaf97..540b49d7 100755 --- a/int_test.py +++ b/int_test.py @@ -2,16 +2,21 @@ import os import sys -import StringIO import unittest import multiprocessing import tempfile import artifactory -import ConfigParser +if sys.version_info[0] < 3: + import StringIO as io + import ConfigParser as configparser +else: + import io + import configparser -config = ConfigParser.ConfigParser() + +config = configparser.ConfigParser() config.read("test.cfg") @@ -133,7 +138,7 @@ def test_deploy(self): if p.exists(): p.unlink() - s = StringIO.StringIO() + s = io.StringIO() s.write("Some test string") p.deploy(s) @@ -186,7 +191,7 @@ def test_open(self): if p.exists(): p.rmdir() - s = StringIO.StringIO() + s = io.StringIO() s.write("Some test string") p.deploy(s) From 6993e276dde05172d5b5f6002eb77c4739b8a74c Mon Sep 17 00:00:00 2001 From: Andrei Scopenco Date: Sun, 23 Oct 2016 12:28:10 +0300 Subject: [PATCH 03/21] Resolve base pep8 warnings --- artifactory.py | 12 ++++++------ int_test.py | 2 +- setup.py | 1 + test.py | 15 ++++++--------- 4 files changed, 14 insertions(+), 16 deletions(-) diff --git a/artifactory.py b/artifactory.py index 93e9ddd8..ba44a893 100755 --- a/artifactory.py +++ b/artifactory.py @@ -83,7 +83,6 @@ def read_config(config_path=default_config_path): verify = p.getboolean(section, 'verify') if p.has_option(section, 'verify') else True cert = p.get(section, 'cert') if p.has_option(section, 'cert') else None - result[section] = {'username': username, 'password': password, 'verify': verify, @@ -121,6 +120,7 @@ def without_http_prefix(url): return url[8:] return url + def get_base_url(config, url): """ Look through config and try to find best matching base for 'url' @@ -141,6 +141,7 @@ def get_base_url(config, url): if without_http_prefix(url).startswith(without_http_prefix(item)): return item + def get_config_entry(config, url): """ Look through config and try to find best matching entry for 'url' @@ -170,6 +171,7 @@ def get_global_config_entry(url): read_global_config() return get_config_entry(global_config, url) + def get_global_base_url(url): """ Look through global config and try to find best matching base for 'url' @@ -753,7 +755,6 @@ def get_properties(self, pathobj): return json.loads(text)['properties'] - def set_properties(self, pathobj, props, recursive): """ Set artifact properties @@ -762,7 +763,7 @@ def set_properties(self, pathobj, props, recursive): 'api/storage', str(pathobj.relative_to(pathobj.drive)).strip('/')]) - params = { 'properties': encode_properties(props) } + params = {'properties': encode_properties(props)} if not recursive: params['recursive'] = '0' @@ -778,7 +779,6 @@ def set_properties(self, pathobj, props, recursive): if code != 204: raise RuntimeError(text) - def del_properties(self, pathobj, props, recursive): """ Delete artifact properties @@ -790,7 +790,7 @@ def del_properties(self, pathobj, props, recursive): 'api/storage', str(pathobj.relative_to(pathobj.drive)).strip('/')]) - params = { 'properties': ','.join(sorted(props)) } + params = {'properties': ','.join(sorted(props))} if not recursive: params['recursive'] = '0' @@ -884,7 +884,7 @@ def __new__(cls, *args, **kwargs): return obj def _init(self, *args, **kwargs): - if not 'template' in kwargs: + if 'template' not in kwargs: kwargs['template'] = _FakePathTemplate(_artifactory_accessor) super(ArtifactoryPath, self)._init(*args, **kwargs) diff --git a/int_test.py b/int_test.py index 540b49d7..16e36bea 100755 --- a/int_test.py +++ b/int_test.py @@ -25,6 +25,7 @@ art_password = config.get("artifactory", "password") art_auth = (art_username, art_password) + class ArtifactoryPathTest(unittest.TestCase): cls = artifactory.ArtifactoryPath @@ -159,7 +160,6 @@ def test_deploy(self): p.unlink() p2.unlink() - def test_deploy_file(self): P = self.cls diff --git a/setup.py b/setup.py index 30c54586..f63921e2 100755 --- a/setup.py +++ b/setup.py @@ -9,6 +9,7 @@ except ImportError: from distutils.core import setup + # PyPi RST variant doesn't understand the 'code' tag. so replacing it # with a regular quote def rst_strip_code_tag(string): diff --git a/test.py b/test.py index 53b58508..3b12f0f9 100755 --- a/test.py +++ b/test.py @@ -112,7 +112,6 @@ def test_splitroot_custom_root(self): check("https://custom/root/foo/baz", ('https://custom/root', '/foo/', 'baz')) - def test_parse_parts(self): check = self._check_parse_parts @@ -120,20 +119,20 @@ def test_parse_parts(self): ('', '', ['.txt'])) check(['http://b/artifactory/c/d.xml'], - ('http://b/artifactory', '/c/', - ['http://b/artifactory/c/', 'd.xml'])) + ('http://b/artifactory', '/c/', + ['http://b/artifactory/c/', 'd.xml'])) check(['http://example.com/artifactory/foo'], ('http://example.com/artifactory', '/foo/', - ['http://example.com/artifactory/foo/'])) + ['http://example.com/artifactory/foo/'])) check(['http://example.com/artifactory/foo/bar'], ('http://example.com/artifactory', '/foo/', - ['http://example.com/artifactory/foo/', 'bar'])) + ['http://example.com/artifactory/foo/', 'bar'])) check(['http://example.com/artifactory/foo/bar/artifactory'], ('http://example.com/artifactory', '/foo/', - ['http://example.com/artifactory/foo/', 'bar', 'artifactory'])) + ['http://example.com/artifactory/foo/', 'bar', 'artifactory'])) class PureArtifactoryPathTest(unittest.TestCase): @@ -148,7 +147,6 @@ def test_root(self): self.assertEqual(P('http://a/artifactory/').root, '') - def test_anchor(self): P = self.cls b = P("http://b/artifactory/c/d.xml") @@ -262,7 +260,6 @@ def test_listdir(self): a.rest_get = MM(return_value=(self.file_stat, 200)) - self.assertRaises(OSError, a.listdir, p) def test_mkdir(self): @@ -348,7 +345,7 @@ def test_artifactory_config(self): self.assertEqual(c['password'], 'ilikerandompasswords') self.assertEqual(c['verify'], False) self.assertEqual(c['cert'], - os.path.expanduser('~/path-to-cert')) + os.path.expanduser('~/path-to-cert')) c = artifactory.get_config_entry(cfg, 'http://bar.net/artifactory') self.assertEqual(c['username'], 'foo') From 1063e2b9c051606e4b7d429bd5be332512a231ee Mon Sep 17 00:00:00 2001 From: Aleksey Burov Date: Thu, 10 Aug 2017 20:26:58 +0700 Subject: [PATCH 04/21] Add requests.session to obj --- artifactory.py | 88 +++++++++++++++++++++++++++----------------------- 1 file changed, 48 insertions(+), 40 deletions(-) diff --git a/artifactory.py b/artifactory.py index ba44a893..5daf39bf 100755 --- a/artifactory.py +++ b/artifactory.py @@ -34,6 +34,7 @@ import json import dateutil.parser import hashlib + try: import requests.packages.urllib3 as urllib3 except ImportError: @@ -43,7 +44,6 @@ except ImportError: import ConfigParser as configparser - default_config_path = '~/.artifactory_python.cfg' global_config = None @@ -220,6 +220,7 @@ class HTTPResponseWrapper(object): since the stream is not rewindable, by the time it tries to send actual content, there is nothing left in the stream. """ + def __init__(self, obj): self.obj = obj @@ -327,10 +328,10 @@ def splitroot(self, part, sep=sep): base = get_global_base_url(part) if base and without_http_prefix(part).startswith(without_http_prefix(base)): - mark = without_http_prefix(base).rstrip(sep)+sep + mark = without_http_prefix(base).rstrip(sep) + sep parts = part.split(mark) else: - mark = sep+'artifactory'+sep + mark = sep + 'artifactory' + sep parts = part.split(mark) if len(parts) >= 2: @@ -409,51 +410,49 @@ class _ArtifactoryAccessor(pathlib._Accessor): """ Implements operations with Artifactory REST API """ - def rest_get(self, url, params=None, headers=None, auth=None, verify=True, cert=None): + + def rest_get(self, url, params=None, headers=None, session=None, verify=True, cert=None): """ - Perform a GET request to url with optional authentication + Perform a GET request to url with requests.session """ - res = requests.get(url, params=params, headers=headers, auth=auth, verify=verify, - cert=cert) + res = session.get(url, params=params, headers=headers, verify=verify, cert=cert) return res.text, res.status_code - def rest_put(self, url, params=None, headers=None, auth=None, verify=True, cert=None): + def rest_put(self, url, params=None, headers=None, session=None, verify=True, cert=None): """ - Perform a PUT request to url with optional authentication + Perform a PUT request to url with requests.session """ - res = requests.put(url, params=params, headers=headers, auth=auth, verify=verify, - cert=cert) + res = session.put(url, params=params, headers=headers, verify=verify, cert=cert) return res.text, res.status_code - def rest_post(self, url, params=None, headers=None, auth=None, verify=True, cert=None): + def rest_post(self, url, params=None, headers=None, session=None, verify=True, cert=None): """ - Perform a PUT request to url with optional authentication + Perform a POST request to url with requests.session """ - res = requests.post(url, params=params, headers=headers, auth=auth, verify=verify, - cert=cert) + res = session.post(url, params=params, headers=headers, verify=verify, cert=cert) return res.text, res.status_code - def rest_del(self, url, params=None, auth=None, verify=True, cert=None): + def rest_del(self, url, params=None, session=None, verify=True, cert=None): """ - Perform a DELETE request to url with optional authentication + Perform a DELETE request to url with requests.session """ - res = requests.delete(url, params=params, auth=auth, verify=verify, cert=cert) + res = session.delete(url, params=params, verify=verify, cert=cert) return res.text, res.status_code - def rest_put_stream(self, url, stream, headers=None, auth=None, verify=True, cert=None): + def rest_put_stream(self, url, stream, headers=None, session=None, verify=True, cert=None): """ - Perform a chunked PUT request to url with optional authentication + Perform a chunked PUT request to url with requests.session This is specifically to upload files. """ - res = requests.put(url, headers=headers, auth=auth, data=stream, verify=verify, cert=cert) + res = session.put(url, headers=headers, data=stream, verify=verify, cert=cert) return res.text, res.status_code - def rest_get_stream(self, url, auth=None, verify=True, cert=None): + def rest_get_stream(self, url, session=None, verify=True, cert=None): """ - Perform a chunked GET request to url with optional authentication + Perform a chunked GET request to url with requests.session This is specifically to download files. """ - res = requests.get(url, auth=auth, stream=True, verify=verify, cert=cert) + res = session.get(url, stream=True, verify=verify, cert=cert) return res.raw, res.status_code def get_stat_json(self, pathobj): @@ -465,7 +464,7 @@ def get_stat_json(self, pathobj): 'api/storage', str(pathobj.relative_to(pathobj.drive)).strip('/')]) - text, code = self.rest_get(url, auth=pathobj.auth, verify=pathobj.verify, + text, code = self.rest_get(url, session=pathobj.session, verify=pathobj.verify, cert=pathobj.cert) if code == 404 and "Unable to find item" in text: raise OSError(2, "No such file or directory: '%s'" % url) @@ -564,8 +563,7 @@ def mkdir(self, pathobj, _): raise OSError(17, "File exists: '%s'" % str(pathobj)) url = str(pathobj) + '/' - text, code = self.rest_put(url, auth=pathobj.auth, verify=pathobj.verify, - cert=pathobj.cert) + text, code = self.rest_put(url, session=pathobj.session, verify=pathobj.verify, cert=pathobj.cert) if not code == 201: raise RuntimeError("%s %d" % (text, code)) @@ -581,8 +579,7 @@ def rmdir(self, pathobj): url = str(pathobj) + '/' - text, code = self.rest_del(url, auth=pathobj.auth, verify=pathobj.verify, - cert=pathobj.cert) + text, code = self.rest_del(url, session=pathobj.session, verify=pathobj.verify, cert=pathobj.cert) if code not in [200, 202, 204]: raise RuntimeError("Failed to delete directory: '%s'" % text) @@ -597,7 +594,7 @@ def unlink(self, pathobj): raise OSError(1, "Operation not permitted: '%s'" % str(pathobj)) url = str(pathobj) - text, code = self.rest_del(url, auth=pathobj.auth, verify=pathobj.verify, + text, code = self.rest_del(url, session=pathobj.session, verify=pathobj.verify, cert=pathobj.cert) if code not in [200, 202, 204]: @@ -614,8 +611,7 @@ def touch(self, pathobj): return url = str(pathobj) - text, code = self.rest_put(url, auth=pathobj.auth, verify=pathobj.verify, - cert=pathobj.cert) + text, code = self.rest_put(url, session=pathobj.session, verify=pathobj.verify, cert=pathobj.cert) if not code == 201: raise RuntimeError("%s %d" % (text, code)) @@ -653,7 +649,7 @@ def open(self, pathobj): seek() """ url = str(pathobj) - raw, code = self.rest_get_stream(url, auth=pathobj.auth, verify=pathobj.verify, + raw, code = self.rest_get_stream(url, session=pathobj.session, verify=pathobj.verify, cert=pathobj.cert) if not code == 200: @@ -684,7 +680,7 @@ def deploy(self, pathobj, fobj, md5=None, sha1=None, parameters=None): text, code = self.rest_put_stream(url, fobj, headers=headers, - auth=pathobj.auth, + session=pathobj.session, verify=pathobj.verify, cert=pathobj.cert) @@ -704,7 +700,7 @@ def copy(self, src, dst, suppress_layouts=False): text, code = self.rest_post(url, params=params, - auth=src.auth, + session=src.session, verify=src.verify, cert=src.cert) @@ -723,7 +719,7 @@ def move(self, src, dst): text, code = self.rest_post(url, params=params, - auth=src.auth, + session=src.session, verify=src.verify, cert=src.cert) @@ -742,7 +738,7 @@ def get_properties(self, pathobj): text, code = self.rest_get(url, params=params, - auth=pathobj.auth, + session=pathobj.session, verify=pathobj.verify, cert=pathobj.cert) @@ -770,7 +766,7 @@ def set_properties(self, pathobj, props, recursive): text, code = self.rest_put(url, params=params, - auth=pathobj.auth, + session=pathobj.session, verify=pathobj.verify, cert=pathobj.cert) @@ -797,7 +793,7 @@ def del_properties(self, pathobj, props, recursive): text, code = self.rest_del(url, params=params, - auth=pathobj.auth, + session=pathobj.session, verify=pathobj.verify, cert=pathobj.cert) @@ -852,7 +848,7 @@ class ArtifactoryPath(pathlib.Path, PureArtifactoryPath): """ # Pathlib limits what members can be present in 'Path' class, # so authentication information has to be added via __slots__ - __slots__ = ('auth', 'verify', 'cert') + __slots__ = ('auth', 'verify', 'cert', 'session') def __new__(cls, *args, **kwargs): """ @@ -881,6 +877,9 @@ def __new__(cls, *args, **kwargs): else: obj.verify = True + obj.session = requests.session() + obj.session.auth = obj.auth + return obj def _init(self, *args, **kwargs): @@ -898,6 +897,7 @@ def parent(self): obj.auth = self.auth obj.verify = self.verify obj.cert = self.cert + obj.session = self.session return obj def with_name(self, name): @@ -908,6 +908,7 @@ def with_name(self, name): obj.auth = self.auth obj.verify = self.verify obj.cert = self.cert + obj.session = self.session return obj def with_suffix(self, suffix): @@ -918,6 +919,7 @@ def with_suffix(self, suffix): obj.auth = self.auth obj.verify = self.verify obj.cert = self.cert + obj.session = self.session return obj def relative_to(self, *other): @@ -930,6 +932,7 @@ def relative_to(self, *other): obj.auth = self.auth obj.verify = self.verify obj.cert = self.cert + obj.session = self.session return obj def joinpath(self, *args): @@ -943,6 +946,7 @@ def joinpath(self, *args): obj.auth = self.auth obj.verify = self.verify obj.cert = self.cert + obj.session = self.session return obj def __truediv__(self, key): @@ -953,6 +957,7 @@ def __truediv__(self, key): obj.auth = self.auth obj.verify = self.verify obj.cert = self.cert + obj.session = self.session return obj def __rtruediv__(self, key): @@ -963,6 +968,7 @@ def __rtruediv__(self, key): obj.auth = self.auth obj.verify = self.verify obj.cert = self.cert + obj.session = self.session return obj if sys.version_info < (3,): @@ -974,6 +980,7 @@ def _make_child(self, args): obj.auth = self.auth obj.verify = self.verify obj.cert = self.cert + obj.session = self.session return obj def _make_child_relpath(self, args): @@ -981,6 +988,7 @@ def _make_child_relpath(self, args): obj.auth = self.auth obj.verify = self.verify obj.cert = self.cert + obj.session = self.session return obj def __iter__(self): From 9213e35b7f922ceed757eeb7e59bf27986da3d5b Mon Sep 17 00:00:00 2001 From: Aleksey Burov Date: Fri, 11 Aug 2017 11:06:43 +0700 Subject: [PATCH 05/21] Fix test with session --- test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.py b/test.py index 3b12f0f9..b6a9688b 100755 --- a/test.py +++ b/test.py @@ -281,7 +281,7 @@ def test_deploy(self): url = "http://b/artifactory/c/d;baz=quux;foo=bar" - a.rest_put_stream.assert_called_with(url, f, headers={}, auth=None, verify=True, cert=None) + a.rest_put_stream.assert_called_with(url, f, headers={}, session=p.session, verify=True, cert=None) class ArtifactoryPathTest(unittest.TestCase): From f06da8311b25fe74aac87f16183175f496a5674a Mon Sep 17 00:00:00 2001 From: Aleksey Burov Date: Fri, 11 Aug 2017 11:31:32 +0700 Subject: [PATCH 06/21] Add support re-use session --- README.md | 21 +++++++++++++++++++++ artifactory.py | 6 ++++-- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b12c0664..208f3605 100644 --- a/README.md +++ b/README.md @@ -79,6 +79,27 @@ path = ArtifactoryPath( path.touch() ``` +## Session ## + +To re-use the established connection, you can pass ```session``` parameter to ArtifactoryPath: + +```python +from artifactory import ArtifactoryPath +import requests +ses = requests.Session() +ses.auth = ('username', 'password') +path = ArtifactoryPath( + "http://my-artifactory/artifactory/myrepo/my-path-1", + sesssion=ses) +path.touch() + +path = ArtifactoryPath( + "http://my-artifactory/artifactory/myrepo/my-path-2", + sesssion=ses) +path.touch() +``` + + ## SSL Cert Verification Options ## See [Requests - SSL verification](http://docs.python-requests.org/en/latest/user/advanced/#ssl-cert-verification) for more details. diff --git a/artifactory.py b/artifactory.py index 5daf39bf..eccb1b22 100755 --- a/artifactory.py +++ b/artifactory.py @@ -863,6 +863,7 @@ def __new__(cls, *args, **kwargs): cfg_entry = get_global_config_entry(obj.drive) obj.auth = kwargs.get('auth', None) obj.cert = kwargs.get('cert', None) + obj.session = kwargs.get('session', None) if obj.auth is None and cfg_entry: obj.auth = (cfg_entry['username'], cfg_entry['password']) @@ -877,8 +878,9 @@ def __new__(cls, *args, **kwargs): else: obj.verify = True - obj.session = requests.session() - obj.session.auth = obj.auth + if obj.session is None: + obj.session = requests.Session() + obj.session.auth = obj.auth return obj From 4294616e8f4f20778d094b9df9bae77a79517834 Mon Sep 17 00:00:00 2001 From: Aleksey Burov Date: Mon, 14 Aug 2017 15:53:11 +0700 Subject: [PATCH 07/21] Fix test - OSError for temp file on Windows system --- test.py | 46 +++++++++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/test.py b/test.py index b6a9688b..d78cc467 100755 --- a/test.py +++ b/test.py @@ -335,30 +335,34 @@ def test_artifactory_config(self): "username=foo\n" + \ "password=bar\n" - with tempfile.NamedTemporaryFile(mode='w+') as tf: + tf = tempfile.NamedTemporaryFile(mode='w+', delete=False) + try: tf.write(cfg) tf.flush() + tf.close() cfg = artifactory.read_config(tf.name) - - c = artifactory.get_config_entry(cfg, 'foo.net/artifactory') - self.assertEqual(c['username'], 'admin') - self.assertEqual(c['password'], 'ilikerandompasswords') - self.assertEqual(c['verify'], False) - self.assertEqual(c['cert'], - os.path.expanduser('~/path-to-cert')) - - c = artifactory.get_config_entry(cfg, 'http://bar.net/artifactory') - self.assertEqual(c['username'], 'foo') - self.assertEqual(c['password'], 'bar') - self.assertEqual(c['verify'], True) - - c = artifactory.get_config_entry(cfg, 'bar.net/artifactory') - self.assertEqual(c['username'], 'foo') - self.assertEqual(c['password'], 'bar') - - c = artifactory.get_config_entry(cfg, 'https://bar.net/artifactory') - self.assertEqual(c['username'], 'foo') - self.assertEqual(c['password'], 'bar') + finally: + os.remove(tf.name) + + c = artifactory.get_config_entry(cfg, 'foo.net/artifactory') + self.assertEqual(c['username'], 'admin') + self.assertEqual(c['password'], 'ilikerandompasswords') + self.assertEqual(c['verify'], False) + self.assertEqual(c['cert'], + os.path.expanduser('~/path-to-cert')) + + c = artifactory.get_config_entry(cfg, 'http://bar.net/artifactory') + self.assertEqual(c['username'], 'foo') + self.assertEqual(c['password'], 'bar') + self.assertEqual(c['verify'], True) + + c = artifactory.get_config_entry(cfg, 'bar.net/artifactory') + self.assertEqual(c['username'], 'foo') + self.assertEqual(c['password'], 'bar') + + c = artifactory.get_config_entry(cfg, 'https://bar.net/artifactory') + self.assertEqual(c['username'], 'foo') + self.assertEqual(c['password'], 'bar') if __name__ == '__main__': From edfa3cbd912246088f0bd98123c0fc95b59b835c Mon Sep 17 00:00:00 2001 From: Aleksey Burov Date: Tue, 15 Aug 2017 16:53:01 +0700 Subject: [PATCH 08/21] init artifactory aql support --- artifactory_aql.py | 43 ++++++++++++++++++++++++++++++++++++ test.py | 55 ++++++++++++++++++++++++++++++++++------------ 2 files changed, 84 insertions(+), 14 deletions(-) create mode 100644 artifactory_aql.py diff --git a/artifactory_aql.py b/artifactory_aql.py new file mode 100644 index 00000000..7f467bdb --- /dev/null +++ b/artifactory_aql.py @@ -0,0 +1,43 @@ +from artifactory import get_global_config_entry +import json +import requests + + +class ArtifactoryAQL(object): + def __init__(self, server, *args, **kwargs): + cfg_entry = get_global_config_entry(server) + self.auth = kwargs.get('auth', None) + self.cert = kwargs.get('cert', None) + self.session = kwargs.get('session', None) + self.server = server + + if self.auth is None and cfg_entry: + self.auth = (cfg_entry['username'], cfg_entry['password']) + + if self.cert is None and cfg_entry: + self.cert = cfg_entry['cert'] + + if 'verify' in kwargs: + self.verify = kwargs.get('verify') + elif cfg_entry: + self.verify = cfg_entry['verify'] + else: + self.verify = True + + if self.session is None: + self.session = requests.Session() + self.session.auth = self.auth + + def send_aql(self, *args): + pass + + @staticmethod + def create_aql_text(*args): + aql_query_text = "" + for arg in args: + if isinstance(arg, dict): + arg = "({})".format(json.dumps(arg)) + elif isinstance(arg, list): + arg = "({})".format(json.dumps(arg)).replace("[", "").replace("]", "") + aql_query_text += arg + return aql_query_text diff --git a/test.py b/test.py index d78cc467..8a6414fd 100755 --- a/test.py +++ b/test.py @@ -1,20 +1,16 @@ #!/usr/bin/env python -import os -import sys import io - -import unittest -import multiprocessing +import os import tempfile -import artifactory -import json -import requests -import datetime -import dateutil +import unittest +import dateutil from mock import MagicMock as MM +import artifactory +from artifactory_aql import ArtifactoryAQL + class UtilTest(unittest.TestCase): def test_matrix_encode(self): @@ -120,19 +116,19 @@ def test_parse_parts(self): check(['http://b/artifactory/c/d.xml'], ('http://b/artifactory', '/c/', - ['http://b/artifactory/c/', 'd.xml'])) + ['http://b/artifactory/c/', 'd.xml'])) check(['http://example.com/artifactory/foo'], ('http://example.com/artifactory', '/foo/', - ['http://example.com/artifactory/foo/'])) + ['http://example.com/artifactory/foo/'])) check(['http://example.com/artifactory/foo/bar'], ('http://example.com/artifactory', '/foo/', - ['http://example.com/artifactory/foo/', 'bar'])) + ['http://example.com/artifactory/foo/', 'bar'])) check(['http://example.com/artifactory/foo/bar/artifactory'], ('http://example.com/artifactory', '/foo/', - ['http://example.com/artifactory/foo/', 'bar', 'artifactory'])) + ['http://example.com/artifactory/foo/', 'bar', 'artifactory'])) class PureArtifactoryPathTest(unittest.TestCase): @@ -365,5 +361,36 @@ def test_artifactory_config(self): self.assertEqual(c['password'], 'bar') +class TestArtifactoryAql(unittest.TestCase): + def setUp(self): + self.aql = ArtifactoryAQL("") + + def test_create_aql_text_simple(self): + args = ["items.find", {"repo": "myrepo"}] + aql_text = self.aql.create_aql_text(*args) + assert aql_text == 'items.find({"repo": "myrepo"})' + + def test_create_aql_text_list(self): + args = ["items.find()", ".include", ["name", "repo"]] + aql_text = self.aql.create_aql_text(*args) + assert aql_text == 'items.find().include("name", "repo")' + + def test_create_aql_text_list_in_dict(self): + args = ["items.find", {"$and": [ + { + "repo": {"$eq": "repo"} + }, + { + "$or": [ + {"path": {"$match": "*path1"}}, + {"path": {"$match": "*path2"}}, + ] + }, + ] + }] + aql_text = self.aql.create_aql_text(*args) + assert aql_text == 'items.find({"$and": [{"repo": {"$eq": "repo"}}, {"$or": [{"path": {"$match": "*path1"}}, {"path": {"$match": "*path2"}}]}]})' + + if __name__ == '__main__': unittest.main() From bc6c249f6a73d98aa0cc6c12401f42fb27fa34a3 Mon Sep 17 00:00:00 2001 From: Aleksey Burov Date: Tue, 15 Aug 2017 16:55:41 +0700 Subject: [PATCH 09/21] implement send_aql method --- artifactory_aql.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/artifactory_aql.py b/artifactory_aql.py index 7f467bdb..040b8bd6 100644 --- a/artifactory_aql.py +++ b/artifactory_aql.py @@ -29,7 +29,12 @@ def __init__(self, server, *args, **kwargs): self.session.auth = self.auth def send_aql(self, *args): - pass + aql_query_url = '{}/api/search/aql'.format(self.server) + aql_query_text = self.create_aql_text(*args) + r = self.session.post(aql_query_url, data=aql_query_text) + r.raise_for_status() + content = r.json() + return content['results'] @staticmethod def create_aql_text(*args): From 80788b89f5d722f8f85d007a64ac67e1860ab016 Mon Sep 17 00:00:00 2001 From: Aleksey Burov Date: Wed, 16 Aug 2017 10:15:14 +0700 Subject: [PATCH 10/21] 2.6 is old python --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index fe0a0040..f40df89b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,6 @@ language: python python: - "3.3" - "2.7" - - "2.6" install: pip install -r requirements.txt script: python test.py From fe5a191afdf8567ab110b11330e89e654565000b Mon Sep 17 00:00:00 2001 From: Aleksey Burov Date: Wed, 16 Aug 2017 10:17:36 +0700 Subject: [PATCH 11/21] add readme AQL --- README.md | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 208f3605..ad091282 100644 --- a/README.md +++ b/README.md @@ -157,4 +157,37 @@ password = @dmin cert = ~/mycert ``` -Whether or not you specify ```http://``` or ```https://``` prefix is not essential. The module will first try to locate the best match and then try to match URLs without prefixes. So if in the config you specify ```https://my-instance.local``` and call ```ArtifactoryPath``` with ```http://my-instance.local```, it will still do the right thing. \ No newline at end of file +Whether or not you specify ```http://``` or ```https://``` prefix is not essential. The module will first try to locate the best match and then try to match URLs without prefixes. So if in the config you specify ```https://my-instance.local``` and call ```ArtifactoryPath``` with ```http://my-instance.local```, it will still do the right thing. + +# Artifactory AQL # + +Supported [Artifactory-AQL](https://www.jfrog.com/confluence/display/RTF/Artifactory+Query+Language) + +```python +from artifactory_aql import ArtifactoryAQL +aql = ArtifactoryAQL( + "http://my-artifactory/artifactory/myrepo/my-path-1") + +# dict support +artifacts = aql.send_aql("items.find", {"repo": "myrepo"}) # send query: items.find({"repo": "myrepo"}) + +# list support +artifacts = aql.send_aql("items.find()", ".include", ["name", "repo"]) # send query: items.find().include("name", "repo") + +# support complex query +args = ["items.find", {"$and": [ + { + "repo": {"$eq": "repo"} + }, + { + "$or": [ + {"path": {"$match": "*path1"}}, + {"path": {"$match": "*path2"}}, + ] + }, +] +}] +artifacts = aql.send_aql(*args) +# send query: +# items.find({"$and": [{"repo": {"$eq": "repo"}}, {"$or": [{"path": {"$match": "*path1"}}, {"path": {"$match": "*path2"}}]}]}) +``` From f456e9588b911521f61d75415af7bf50604ed9f5 Mon Sep 17 00:00:00 2001 From: Aleksey Burov Date: Wed, 16 Aug 2017 10:36:41 +0700 Subject: [PATCH 12/21] Move aql to ArtifactoryPath --- README.md | 11 +++++------ artifactory.py | 33 ++++++++++++++++++++++++------- artifactory_aql.py | 48 ---------------------------------------------- test.py | 4 +--- 4 files changed, 32 insertions(+), 64 deletions(-) delete mode 100644 artifactory_aql.py diff --git a/README.md b/README.md index ad091282..ffc8d9f0 100644 --- a/README.md +++ b/README.md @@ -164,15 +164,14 @@ Whether or not you specify ```http://``` or ```https://``` prefix is not essenti Supported [Artifactory-AQL](https://www.jfrog.com/confluence/display/RTF/Artifactory+Query+Language) ```python -from artifactory_aql import ArtifactoryAQL -aql = ArtifactoryAQL( - "http://my-artifactory/artifactory/myrepo/my-path-1") +from artifactory import ArtifactoryPath +aql = ArtifactoryPath( "http://my-artifactory/artifactory") # path to artifactory, NO repo # dict support -artifacts = aql.send_aql("items.find", {"repo": "myrepo"}) # send query: items.find({"repo": "myrepo"}) +artifacts = aql.aql("items.find", {"repo": "myrepo"}) # send query: items.find({"repo": "myrepo"}) # list support -artifacts = aql.send_aql("items.find()", ".include", ["name", "repo"]) # send query: items.find().include("name", "repo") +artifacts = aql.aql("items.find()", ".include", ["name", "repo"]) # send query: items.find().include("name", "repo") # support complex query args = ["items.find", {"$and": [ @@ -187,7 +186,7 @@ args = ["items.find", {"$and": [ }, ] }] -artifacts = aql.send_aql(*args) +artifacts_list = aql.aql(*args) # send query: # items.find({"$and": [{"repo": {"$eq": "repo"}}, {"$or": [{"path": {"$match": "*path1"}}, {"path": {"$match": "*path2"}}]}]}) ``` diff --git a/artifactory.py b/artifactory.py index eccb1b22..b9fa93b5 100755 --- a/artifactory.py +++ b/artifactory.py @@ -24,16 +24,16 @@ pure paths can be used. """ -import os -import sys -import errno -import pathlib import collections -import requests -import re +import errno +import hashlib import json +import os +import pathlib +import sys + import dateutil.parser -import hashlib +import requests try: import requests.packages.urllib3 as urllib3 @@ -1267,6 +1267,25 @@ def del_properties(self, properties, recursive=None): """ return self._accessor.del_properties(self, properties, recursive) + def aql(self, *args): + aql_query_url = '{}/api/search/aql'.format(self.drive) + aql_query_text = self.create_aql_text(*args) + r = self.session.post(aql_query_url, data=aql_query_text) + r.raise_for_status() + content = r.json() + return content['results'] + + @staticmethod + def create_aql_text(*args): + aql_query_text = "" + for arg in args: + if isinstance(arg, dict): + arg = "({})".format(json.dumps(arg)) + elif isinstance(arg, list): + arg = "({})".format(json.dumps(arg)).replace("[", "").replace("]", "") + aql_query_text += arg + return aql_query_text + def walk(pathobj, topdown=True): """ diff --git a/artifactory_aql.py b/artifactory_aql.py deleted file mode 100644 index 040b8bd6..00000000 --- a/artifactory_aql.py +++ /dev/null @@ -1,48 +0,0 @@ -from artifactory import get_global_config_entry -import json -import requests - - -class ArtifactoryAQL(object): - def __init__(self, server, *args, **kwargs): - cfg_entry = get_global_config_entry(server) - self.auth = kwargs.get('auth', None) - self.cert = kwargs.get('cert', None) - self.session = kwargs.get('session', None) - self.server = server - - if self.auth is None and cfg_entry: - self.auth = (cfg_entry['username'], cfg_entry['password']) - - if self.cert is None and cfg_entry: - self.cert = cfg_entry['cert'] - - if 'verify' in kwargs: - self.verify = kwargs.get('verify') - elif cfg_entry: - self.verify = cfg_entry['verify'] - else: - self.verify = True - - if self.session is None: - self.session = requests.Session() - self.session.auth = self.auth - - def send_aql(self, *args): - aql_query_url = '{}/api/search/aql'.format(self.server) - aql_query_text = self.create_aql_text(*args) - r = self.session.post(aql_query_url, data=aql_query_text) - r.raise_for_status() - content = r.json() - return content['results'] - - @staticmethod - def create_aql_text(*args): - aql_query_text = "" - for arg in args: - if isinstance(arg, dict): - arg = "({})".format(json.dumps(arg)) - elif isinstance(arg, list): - arg = "({})".format(json.dumps(arg)).replace("[", "").replace("]", "") - aql_query_text += arg - return aql_query_text diff --git a/test.py b/test.py index 8a6414fd..57ef591d 100755 --- a/test.py +++ b/test.py @@ -9,8 +9,6 @@ from mock import MagicMock as MM import artifactory -from artifactory_aql import ArtifactoryAQL - class UtilTest(unittest.TestCase): def test_matrix_encode(self): @@ -363,7 +361,7 @@ def test_artifactory_config(self): class TestArtifactoryAql(unittest.TestCase): def setUp(self): - self.aql = ArtifactoryAQL("") + self.aql = artifactory.ArtifactoryPath("") def test_create_aql_text_simple(self): args = ["items.find", {"repo": "myrepo"}] From efbca5a3383250e667c95df80032c1397ff5132b Mon Sep 17 00:00:00 2001 From: Aleksey Burov Date: Wed, 16 Aug 2017 10:58:20 +0700 Subject: [PATCH 13/21] fixed https://github.com/devopshq/artifactory/issues/3 - Full support AQL and conver from AQL dict to Pathlib object --- README.md | 8 +++++++- artifactory.py | 22 ++++++++++++++++++++++ test.py | 15 ++++++++++++++- 3 files changed, 43 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ffc8d9f0..50c318db 100644 --- a/README.md +++ b/README.md @@ -186,7 +186,13 @@ args = ["items.find", {"$and": [ }, ] }] -artifacts_list = aql.aql(*args) + # send query: # items.find({"$and": [{"repo": {"$eq": "repo"}}, {"$or": [{"path": {"$match": "*path1"}}, {"path": {"$match": "*path2"}}]}]}) +# artifacts_list contains raw data (list of dict) +artifacts_list = aql.aql(*args) + +# You can convert to patlib object: +artifact_pathlib = map(aql.from_aql, artifacts_list) +artifact_pathlib_list = list(map(aql.from_aql, artifacts_list)) ``` diff --git a/artifactory.py b/artifactory.py index b9fa93b5..8caeea80 100755 --- a/artifactory.py +++ b/artifactory.py @@ -1268,6 +1268,11 @@ def del_properties(self, properties, recursive=None): return self._accessor.del_properties(self, properties, recursive) def aql(self, *args): + """ + Send AQL query to Artifactory + :param args: + :return: + """ aql_query_url = '{}/api/search/aql'.format(self.drive) aql_query_text = self.create_aql_text(*args) r = self.session.post(aql_query_url, data=aql_query_text) @@ -1277,6 +1282,9 @@ def aql(self, *args): @staticmethod def create_aql_text(*args): + """ + Create AQL querty from string\list\dict arguments + """ aql_query_text = "" for arg in args: if isinstance(arg, dict): @@ -1286,6 +1294,20 @@ def create_aql_text(*args): aql_query_text += arg return aql_query_text + def from_aql(self, result): + """ + Convert raw AQL result to pathlib object + :param result: ONE raw result + :return: + """ + result_type = result.get('type') + if result_type not in ('file', 'folder'): + raise RuntimeError("Path object with type '{}' doesn't support. File or folder only".format(result_type)) + + result_path = "{}/{repo}/{path}/{name}".format(self.drive, **result) + obj = ArtifactoryPath(result_path, auth=self.auth, verify=self.verify, cert=self.cert, session=self.session) + return obj + def walk(pathobj, topdown=True): """ diff --git a/test.py b/test.py index 57ef591d..05dcb5e6 100755 --- a/test.py +++ b/test.py @@ -10,6 +10,7 @@ import artifactory + class UtilTest(unittest.TestCase): def test_matrix_encode(self): params = {"foo": "bar", @@ -361,7 +362,7 @@ def test_artifactory_config(self): class TestArtifactoryAql(unittest.TestCase): def setUp(self): - self.aql = artifactory.ArtifactoryPath("") + self.aql = artifactory.ArtifactoryPath("http://b/artifactory") def test_create_aql_text_simple(self): args = ["items.find", {"repo": "myrepo"}] @@ -389,6 +390,18 @@ def test_create_aql_text_list_in_dict(self): aql_text = self.aql.create_aql_text(*args) assert aql_text == 'items.find({"$and": [{"repo": {"$eq": "repo"}}, {"$or": [{"path": {"$match": "*path1"}}, {"path": {"$match": "*path2"}}]}]})' + def test_from_aql_file(self): + result = { + 'repo': 'reponame', + 'path': 'folder1/folder2', + 'name': 'name.nupkg', + 'type': 'file', + } + artifact = self.aql.from_aql(result) + assert artifact.drive == "http://b/artifactory" + assert artifact.name == "name.nupkg" + assert artifact.root == "/reponame/" + if __name__ == '__main__': unittest.main() From fe7e43a87e06cf301184438d4a37daf44f05d77d Mon Sep 17 00:00:00 2001 From: Timur Gilmullin Date: Tue, 26 Dec 2017 15:18:06 +0300 Subject: [PATCH 14/21] theme: cayman --- _config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_config.yml b/_config.yml index fc24e7a6..ccbddf08 100644 --- a/_config.yml +++ b/_config.yml @@ -1 +1 @@ -theme: jekyll-theme-hacker \ No newline at end of file +theme: cayman \ No newline at end of file From 834aeee58744f52dc8afc60e1c3835407bd6c7c1 Mon Sep 17 00:00:00 2001 From: Timur Gilmullin Date: Tue, 26 Dec 2017 15:28:23 +0300 Subject: [PATCH 15/21] setup.py refactoring --- setup.py | 46 ++++++++-------------------------------------- 1 file changed, 8 insertions(+), 38 deletions(-) diff --git a/setup.py b/setup.py index d8cbb073..10961225 100755 --- a/setup.py +++ b/setup.py @@ -5,6 +5,10 @@ import os import re import sys +try: + from setuptools import setup +except ImportError: + from distutils.core import setup __version__ = '0.2' # identify main version of dohq-artifactory @@ -25,49 +29,15 @@ print("dohq-artifactory build version = {}".format(__version__)) - -try: - from setuptools import setup -except ImportError: - from distutils.core import setup - - -# PyPi RST variant doesn't understand the 'code' tag. so replacing it -# with a regular quote -def rst_strip_code_tag(string): - return re.sub('^\\.\\. code:: .*', '::', string, flags=re.MULTILINE) - - -# Utility function to read the README file. -# To upload to PyPi, you need to have 'pypandoc'. -# Otherwise the readme will be clumsy. -def convert_rst(): - return lambda fname: rst_strip_code_tag( - convert(os.path.join(os.path.dirname(__file__), fname), 'rst')) - - -def read_md(): - return lambda fname: open(os.path.join(os.path.dirname(__file__), fname), 'r').read() - - -try: - from pypandoc import convert - read_md = convert_rst() - -except ImportError: - print("warning: pypandoc module not found, could not convert Markdown to RST") - read_md = read_md() - - setup( name='dohq-artifactory', version=__version__, py_modules=['artifactory'], license='MIT License', - description='A Python to Artifactory interface', - long_description=read_md('README.md'), - author='Konstantin Nazarov', - author_email='knazarov@parallels.com', + description='A Python interface to Artifactory', + long_description='See full documentation here: https://devopshq.github.io/artifactory/', + author='Alexey Burov', + author_email='aburov@ptsecurity.com', classifiers=[ 'Development Status :: 3 - Alpha', 'Intended Audience :: Developers', From cd048be31d9e67fc387a7bd6ca39bf6535cd4d22 Mon Sep 17 00:00:00 2001 From: Timur Gilmullin Date: Tue, 26 Dec 2017 15:32:05 +0300 Subject: [PATCH 16/21] unused imports are removed from setup.py --- setup.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/setup.py b/setup.py index 10961225..73aa5881 100755 --- a/setup.py +++ b/setup.py @@ -3,8 +3,6 @@ import os -import re -import sys try: from setuptools import setup except ImportError: From 6e2758fbb7e6c65ad0d4ad703762f74d925ba5ff Mon Sep 17 00:00:00 2001 From: Timur Gilmullin Date: Tue, 26 Dec 2017 15:33:18 +0300 Subject: [PATCH 17/21] Set theme jekyll-theme-cayman --- _config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_config.yml b/_config.yml index ccbddf08..c4192631 100644 --- a/_config.yml +++ b/_config.yml @@ -1 +1 @@ -theme: cayman \ No newline at end of file +theme: jekyll-theme-cayman \ No newline at end of file From b4ff6e03005474c26f9310f112b2f6c3148a36c0 Mon Sep 17 00:00:00 2001 From: Timur Gilmullin Date: Tue, 26 Dec 2017 15:42:16 +0300 Subject: [PATCH 18/21] travis script is fixed --- .travis.yml | 5 ----- requirements.txt | 6 ------ 2 files changed, 11 deletions(-) delete mode 100644 requirements.txt diff --git a/.travis.yml b/.travis.yml index 98d11326..b9fb53d5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,11 +2,6 @@ language: python python: - '3.5' - '2.7' -before_install: -- sudo apt-get install pandoc -install: -- pip install -r requirements.txt -- pip install pypandoc script: python test.py deploy: provider: pypi diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index c3e0d51f..00000000 --- a/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -requests ---index-url https://pypi.python.org/simple/ - -pypandoc - --e . From 076e2e67e322f1b3f3641a6aeb5dab0d09b06121 Mon Sep 17 00:00:00 2001 From: Timur Gilmullin Date: Tue, 26 Dec 2017 15:47:10 +0300 Subject: [PATCH 19/21] python-dateutil is added --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index b9fb53d5..4b7b35d8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,8 @@ language: python python: - '3.5' - '2.7' +install: +- pip install python-dateutil script: python test.py deploy: provider: pypi From d2c7619229831614be9ef94754761fd91e56e963 Mon Sep 17 00:00:00 2001 From: Timur Gilmullin Date: Tue, 26 Dec 2017 15:49:43 +0300 Subject: [PATCH 20/21] requests is added --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4b7b35d8..b570bd1b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,7 @@ python: - '3.5' - '2.7' install: -- pip install python-dateutil +- pip install python-dateutil requests script: python test.py deploy: provider: pypi From 6720c8c7886c58881031ee52b293b83cda9dbd08 Mon Sep 17 00:00:00 2001 From: Timur Gilmullin Date: Tue, 26 Dec 2017 15:51:05 +0300 Subject: [PATCH 21/21] pathlib is added --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index b570bd1b..ef1b03c9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,7 @@ python: - '3.5' - '2.7' install: -- pip install python-dateutil requests +- pip install python-dateutil requests pathlib script: python test.py deploy: provider: pypi