From 70cab111cb5e8d8232a0350f5eaccf900775ad96 Mon Sep 17 00:00:00 2001 From: Arthur Flam Date: Mon, 17 Feb 2020 11:54:13 +0200 Subject: [PATCH] Change `import qatools` to `import qaboard`, while staying backward compatible --- MANIFEST.in | 4 +-- qaboard/__init__.py | 9 +++++++ qaboard/__main__.py | 2 ++ {qatools => qaboard}/api.py | 0 {qatools => qaboard}/bit_accuracy.py | 0 {qatools => qaboard}/check_for_updates.py | 8 +++--- {qatools => qaboard}/ci_helpers.py | 0 {qatools => qaboard}/config.py | 0 {qatools => qaboard}/conventions.py | 0 {qatools => qaboard}/gitlab.py | 0 {qatools => qaboard}/init.py | 2 +- {qatools => qaboard}/iterators.py | 0 {qatools => qaboard}/lsf.py | 0 qatools/qatools.py => qaboard/qa.py | 22 +++++++-------- {qatools => qaboard}/qa_aggregate.py | 2 +- .../sample_project/.gitignore | 0 .../sample_project/.gitlab-ci.yml | 0 {qatools => qaboard}/sample_project/README.md | 0 .../sample_project/cli_tests/a.txt | 0 .../sample_project/cli_tests/b.txt | 0 .../sample_project/cli_tests/dir/c.txt | 0 .../sample_project/qa/batches.yaml | 0 .../sample_project/qa/main.py | 0 .../sample_project/qa/metrics.yaml | 0 .../sample_project/qaboard.yaml | 0 {qatools => qaboard}/search.py | 0 {qatools => qaboard}/tuning.py | 0 {qatools => qaboard}/utils.py | 0 qatools/__init__.py | 27 +++++++++++++------ qatools/__main__.py | 2 -- setup.py | 23 +++++++++++----- tests/test_conventions.py | 4 +-- tests/test_iterators.py | 4 +-- website/docusaurus.config.js | 9 +++---- 34 files changed, 74 insertions(+), 44 deletions(-) create mode 100755 qaboard/__init__.py create mode 100755 qaboard/__main__.py rename {qatools => qaboard}/api.py (100%) rename {qatools => qaboard}/bit_accuracy.py (100%) rename {qatools => qaboard}/check_for_updates.py (81%) rename {qatools => qaboard}/ci_helpers.py (100%) rename {qatools => qaboard}/config.py (100%) rename {qatools => qaboard}/conventions.py (100%) rename {qatools => qaboard}/gitlab.py (100%) rename {qatools => qaboard}/init.py (94%) rename {qatools => qaboard}/iterators.py (100%) rename {qatools => qaboard}/lsf.py (100%) rename qatools/qatools.py => qaboard/qa.py (99%) rename {qatools => qaboard}/qa_aggregate.py (97%) rename {qatools => qaboard}/sample_project/.gitignore (100%) rename {qatools => qaboard}/sample_project/.gitlab-ci.yml (100%) rename {qatools => qaboard}/sample_project/README.md (100%) rename {qatools => qaboard}/sample_project/cli_tests/a.txt (100%) rename {qatools => qaboard}/sample_project/cli_tests/b.txt (100%) rename {qatools => qaboard}/sample_project/cli_tests/dir/c.txt (100%) rename {qatools => qaboard}/sample_project/qa/batches.yaml (100%) rename {qatools => qaboard}/sample_project/qa/main.py (100%) rename {qatools => qaboard}/sample_project/qa/metrics.yaml (100%) rename {qatools => qaboard}/sample_project/qaboard.yaml (100%) rename {qatools => qaboard}/search.py (100%) rename {qatools => qaboard}/tuning.py (100%) rename {qatools => qaboard}/utils.py (100%) mode change 100644 => 100755 qatools/__init__.py delete mode 100644 qatools/__main__.py diff --git a/MANIFEST.in b/MANIFEST.in index 133bc743..27b3a052 100755 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,3 @@ -recursive-include qatools/sample_project/qa * -include qatools/sample_project/qaboard.yaml +recursive-include qaboard/sample_project/qa * +include qaboard/sample_project/qaboard.yaml diff --git a/qaboard/__init__.py b/qaboard/__init__.py new file mode 100755 index 00000000..a910a2c6 --- /dev/null +++ b/qaboard/__init__.py @@ -0,0 +1,9 @@ +# need to be update setup.py as well +__version__ = '0.8.2' +from .check_for_updates import check_for_updates +check_for_updates() + +from .config import on_windows, on_linux, on_lsf, on_vdi, is_ci +from .config import config, merge +from .conventions import slugify +from .qa import qa diff --git a/qaboard/__main__.py b/qaboard/__main__.py new file mode 100755 index 00000000..29be219f --- /dev/null +++ b/qaboard/__main__.py @@ -0,0 +1,2 @@ +from .qa import main +main() diff --git a/qatools/api.py b/qaboard/api.py similarity index 100% rename from qatools/api.py rename to qaboard/api.py diff --git a/qatools/bit_accuracy.py b/qaboard/bit_accuracy.py similarity index 100% rename from qatools/bit_accuracy.py rename to qaboard/bit_accuracy.py diff --git a/qatools/check_for_updates.py b/qaboard/check_for_updates.py similarity index 81% rename from qatools/check_for_updates.py rename to qaboard/check_for_updates.py index 0dd832c0..79dd0953 100755 --- a/qatools/check_for_updates.py +++ b/qaboard/check_for_updates.py @@ -20,7 +20,7 @@ def latest_qatools_version(): import re try: # we surely could do something more robust - r = requests.get('http://gitlab-srv/common-infrastructure/qatools/raw/master/setup.py', timeout=1) + r = requests.get('https://raw.githubusercontent.com/Samsung/qaboard/master/setup.py', timeout=1) except: return None for l in r.text.split('\n'): @@ -71,9 +71,9 @@ def check_for_updates(): if latest_version: - from qatools import __version__ as current_version + from qaboard import __version__ as current_version to_ints = lambda v: [int(n) for n in v.split('.')] newer_version_available = to_ints(current_version) < to_ints(latest_version) if newer_version_available: - click.secho(f'A new version of qatools is available! Upgrade to {latest_version}:', fg='yellow', bold=True, err=True) - click.secho('$ pip install --upgrade --trusted-host pypi.python.org --trusted-host pypi.org --trusted-host files.pythonhosted.org git+http://gitlab-srv/common-infrastructure/qatools', fg='yellow', err=True) + click.secho(f'[INFO] A new version of qaboard is available! Upgrade to {latest_version}:', fg='yellow', bold=True, err=True) + click.secho(' $ pip install --upgrade qaboard', fg='yellow', err=True) diff --git a/qatools/ci_helpers.py b/qaboard/ci_helpers.py similarity index 100% rename from qatools/ci_helpers.py rename to qaboard/ci_helpers.py diff --git a/qatools/config.py b/qaboard/config.py similarity index 100% rename from qatools/config.py rename to qaboard/config.py diff --git a/qatools/conventions.py b/qaboard/conventions.py similarity index 100% rename from qatools/conventions.py rename to qaboard/conventions.py diff --git a/qatools/gitlab.py b/qaboard/gitlab.py similarity index 100% rename from qatools/gitlab.py rename to qaboard/gitlab.py diff --git a/qatools/init.py b/qaboard/init.py similarity index 94% rename from qatools/init.py rename to qaboard/init.py index cf7600a8..1ed75ae8 100755 --- a/qatools/init.py +++ b/qaboard/init.py @@ -41,7 +41,7 @@ def qa_init(): pass except: import pkg_resources - qatools_dir = Path(pkg_resources.resource_filename('qatools', '')) + qatools_dir = Path(pkg_resources.resource_filename('qaboard', '')) click.secho('Creating a `qatools` configuration based on the sample project 🎉', fg='green') shutil.copy(str(qatools_dir / 'sample_project/qaboard.yaml'), 'qaboard.yaml') diff --git a/qatools/iterators.py b/qaboard/iterators.py similarity index 100% rename from qatools/iterators.py rename to qaboard/iterators.py diff --git a/qatools/lsf.py b/qaboard/lsf.py similarity index 100% rename from qatools/lsf.py rename to qaboard/lsf.py diff --git a/qatools/qatools.py b/qaboard/qa.py similarity index 99% rename from qatools/qatools.py rename to qaboard/qa.py index d6139c7d..59222fd4 100755 --- a/qatools/qatools.py +++ b/qaboard/qa.py @@ -47,7 +47,7 @@ @click.option('--database', type=PathType(), help="Input database location") @click.option('--type', 'input_type', default=default_input_type, help="How we define inputs") @click.option('--offline', is_flag=True, help="Do not notify QA-Board about run statuses.") -def cli(ctx, platform, configuration, label, tuning, tuning_filepath, dryrun, share, database, input_type, offline): +def qa(ctx, platform, configuration, label, tuning, tuning_filepath, dryrun, share, database, input_type, offline): """Entrypoint to running your algo, launching batchs...""" # We want all paths to be relative to top-most qaboard.yaml # it should be located at the root of the git repository @@ -122,7 +122,7 @@ def cli(ctx, platform, configuration, label, tuning, tuning_filepath, dryrun, sh ctx.obj['color'] = is_ci or share -@cli.command() +@qa.command() @click.option('-i', '--input', 'input_path', type=PathType(), help='Path of the input/recording/test we should work on, relative to the database directory.') @click.option('-o', '--output', 'output_path', type=PathType(), default=None, help='Custom output directory path. If not provided, defaults to ctx.obj["prefix_output_dir"] / input_path.with_suffix('')') @click.argument('variable') @@ -144,7 +144,7 @@ def get(ctx, input_path, output_path, variable): -@cli.command(context_settings=dict( +@qa.command(context_settings=dict( ignore_unknown_options=True, allow_interspersed_args=False, )) @@ -293,7 +293,7 @@ def postprocess_(runtime_metrics, context, skip=False, save_manifests_in_databas -@cli.command(context_settings=dict( +@qa.command(context_settings=dict( ignore_unknown_options=True, )) @click.pass_context @@ -318,7 +318,7 @@ def postprocess(ctx, input_path, output_path, forwarded_args): -@cli.command(context_settings=dict( +@qa.command(context_settings=dict( ignore_unknown_options=True, )) @click.pass_context @@ -341,7 +341,7 @@ def sync(ctx, input_path, output_path): lsf_config = config.get('runners').get('lsf', {}) if 'runners' in config else config.get('lsf', {}) -@cli.command(context_settings=dict( +@qa.command(context_settings=dict( ignore_unknown_options=True, )) @click.option('--batch', '-b', 'batches', multiple=True, help="We run over all inputs+configs+database in those batches") @@ -540,7 +540,7 @@ def batch(ctx, batches, batches_files, tuning_search, tuning_search_file, no_wai -@cli.command() +@qa.command() @click.option('--file', '-f', 'files', multiple=True, help="Save spcific files instead of artifacts indicated by yaml file") # Do we use this? let's deprecate and remove @click.option('--out', '-o', 'artifacts_path', default='', help="Path to save artifacts in case of specified files") @@ -629,7 +629,7 @@ def save_artifacts(ctx, files, artifacts_path, groups): notify_qa_database(object_type='commit', **ctx.obj) -@cli.command() +@qa.command() @click.pass_context @click.option('--batch', '-b', 'batches', required=True, multiple=True, help="Only check bit-accuracy for this batch of inputs+configs+database.") @click.option('--batches-file', 'batches_files', default=default_batches_files, multiple=True, help="YAML file listing batches of inputs+config+database selected from the database.") @@ -681,7 +681,7 @@ def check_bit_accuracy_manifest(ctx, batches, batches_files): -@cli.command() +@qa.command() @click.pass_context @click.option( "--reference", @@ -751,7 +751,7 @@ def check_bit_accuracy(ctx, reference, batches, batches_files, reference_platfor click.secho(f"https://qa/{config['project']['name']}/commit/{commit_id}?reference={reference_commit.hexsha}&selected_views=bit_accuracy", fg='red') exit(1) -@cli.command(context_settings=dict( +@qa.command(context_settings=dict( ignore_unknown_options=True, )) @click.option('--batch', '-b', 'batches', required=True, multiple=True, help="Use the inputs+configs+database in those batches") @@ -853,7 +853,7 @@ def optimize(ctx, batches, batches_files, config_file, forwarded_args): def main(): - cli(obj={}, auto_envvar_prefix='QA') + qa(obj={}, auto_envvar_prefix='QA') if __name__ == '__main__': main() diff --git a/qatools/qa_aggregate.py b/qaboard/qa_aggregate.py similarity index 97% rename from qatools/qa_aggregate.py rename to qaboard/qa_aggregate.py index bf200fce..ab5f79f7 100755 --- a/qatools/qa_aggregate.py +++ b/qaboard/qa_aggregate.py @@ -23,7 +23,7 @@ from pathlib import Path import click -from qatools.config import root_qatools, available_metrics +from qaboard.config import root_qatools, available_metrics diff --git a/qatools/sample_project/.gitignore b/qaboard/sample_project/.gitignore similarity index 100% rename from qatools/sample_project/.gitignore rename to qaboard/sample_project/.gitignore diff --git a/qatools/sample_project/.gitlab-ci.yml b/qaboard/sample_project/.gitlab-ci.yml similarity index 100% rename from qatools/sample_project/.gitlab-ci.yml rename to qaboard/sample_project/.gitlab-ci.yml diff --git a/qatools/sample_project/README.md b/qaboard/sample_project/README.md similarity index 100% rename from qatools/sample_project/README.md rename to qaboard/sample_project/README.md diff --git a/qatools/sample_project/cli_tests/a.txt b/qaboard/sample_project/cli_tests/a.txt similarity index 100% rename from qatools/sample_project/cli_tests/a.txt rename to qaboard/sample_project/cli_tests/a.txt diff --git a/qatools/sample_project/cli_tests/b.txt b/qaboard/sample_project/cli_tests/b.txt similarity index 100% rename from qatools/sample_project/cli_tests/b.txt rename to qaboard/sample_project/cli_tests/b.txt diff --git a/qatools/sample_project/cli_tests/dir/c.txt b/qaboard/sample_project/cli_tests/dir/c.txt similarity index 100% rename from qatools/sample_project/cli_tests/dir/c.txt rename to qaboard/sample_project/cli_tests/dir/c.txt diff --git a/qatools/sample_project/qa/batches.yaml b/qaboard/sample_project/qa/batches.yaml similarity index 100% rename from qatools/sample_project/qa/batches.yaml rename to qaboard/sample_project/qa/batches.yaml diff --git a/qatools/sample_project/qa/main.py b/qaboard/sample_project/qa/main.py similarity index 100% rename from qatools/sample_project/qa/main.py rename to qaboard/sample_project/qa/main.py diff --git a/qatools/sample_project/qa/metrics.yaml b/qaboard/sample_project/qa/metrics.yaml similarity index 100% rename from qatools/sample_project/qa/metrics.yaml rename to qaboard/sample_project/qa/metrics.yaml diff --git a/qatools/sample_project/qaboard.yaml b/qaboard/sample_project/qaboard.yaml similarity index 100% rename from qatools/sample_project/qaboard.yaml rename to qaboard/sample_project/qaboard.yaml diff --git a/qatools/search.py b/qaboard/search.py similarity index 100% rename from qatools/search.py rename to qaboard/search.py diff --git a/qatools/tuning.py b/qaboard/tuning.py similarity index 100% rename from qatools/tuning.py rename to qaboard/tuning.py diff --git a/qatools/utils.py b/qaboard/utils.py similarity index 100% rename from qatools/utils.py rename to qaboard/utils.py diff --git a/qatools/__init__.py b/qatools/__init__.py old mode 100644 new mode 100755 index 49e2cfcd..4e05bee7 --- a/qatools/__init__.py +++ b/qatools/__init__.py @@ -1,9 +1,20 @@ -# need to be update setup.py as well -__version__ = '0.8.2' -from .check_for_updates import check_for_updates -check_for_updates() +""" +For backward-compatibility: once qaboard was imported as "import qatools" +Inspired by https://stackoverflow.com/a/24324577/5993501 +""" +import sys +import qaboard -from .config import on_windows, on_linux, on_lsf, on_vdi, is_ci -from .config import config, merge -from .conventions import slugify -from .qatools import cli +modules = [ + 'qaboard', + 'qaboard.check_for_updates', + 'qaboard.config', + 'qaboard.utils', + 'qaboard.conventions', + 'qaboard.iterators', + 'qaboard.qa', + 'qaboard.lsf', + 'qaboard.api' +] +for m in modules: + sys.modules[m.replace('qaboard', 'qatools')] = sys.modules[m] diff --git a/qatools/__main__.py b/qatools/__main__.py deleted file mode 100644 index 5cfbd53b..00000000 --- a/qatools/__main__.py +++ /dev/null @@ -1,2 +0,0 @@ -from .qatools import main -main() diff --git a/setup.py b/setup.py index dbf6b71d..594ccb19 100755 --- a/setup.py +++ b/setup.py @@ -17,6 +17,18 @@ # (works, ugly) patch: `+ verify=False` in lib/python37/site-packages/twine/repository.py:175 # (helps) ssh $nice_host twine upload --verbose -u __token__ dist/* + + +Until https://github.com/pypa/pip/issues/2195 is resolved, +for local development with `pip install` without `--editable`, +you'll likely want to speed-up pip +``` +# site-packages/pip/_internal/download.py +778: + from shutil import ignore_patterns + + shutil.copytree(link_path, location, symlinks=True, ignore=ignore_patterns('.git', 'node_modules')) + - shutil.copytree(link_path, location, symlinks=True) +``` + """ # https://github.com/ninjaaron/fast-entry_points import fastentrypoints @@ -24,9 +36,8 @@ from pathlib import Path from setuptools import setup, find_packages -# The directory containing this file -HERE = Path(__file__).parent -README = (HERE / "README.md").read_text(encoding='utf-8') +README_md = Path(__file__).parent / "README.md" +README = README_md.read_text(encoding='utf-8') setup( name='qaboard', @@ -52,8 +63,8 @@ "Topic :: Software Development :: Quality Assurance", ], - python_requires='>=3.6', packages=find_packages(exclude=("tests","backend")), + python_requires='>=3.6', install_requires=[ 'dataclasses', # Backport for python3.6 'click>=7.0', # CLI for humans. In v7 they changed CLI command conventions, started using "-" vs "_" @@ -75,7 +86,7 @@ extras_require={ # Optionnal needed only for `qa optimize` # Since its CLI usage is not straightforward, it's best kept at an optionnal dependency - # Enable with `pip install qatools[optimize]` + # Enable with `pip install qaboard[optimize]` # Since the package is not maintained anymore, maybe we should review the optimization loop # and use nevergrad like all the cool kids. 'optimize': ["skopt"], @@ -83,7 +94,7 @@ entry_points={ "console_scripts": [ - 'qa = qatools.qatools:main' + 'qa = qaboard.qa:main' ] }, diff --git a/tests/test_conventions.py b/tests/test_conventions.py index 9a1d8e88..7756c221 100755 --- a/tests/test_conventions.py +++ b/tests/test_conventions.py @@ -7,13 +7,13 @@ class TestConventions(unittest.TestCase): def test_serialize_config(self): - from qatools.conventions import serialize_config + from qaboard.conventions import serialize_config self.assertEqual(serialize_config(['a', 'b', 'c']), 'a:b:c') self.assertEqual(serialize_config(['a', {'b': 1}]), 'a:{"b": 1}') self.assertEqual(serialize_config(['a', '{"b":1']), 'a:{"b":1') def test_deserialize_config(self): - from qatools.conventions import deserialize_config + from qaboard.conventions import deserialize_config self.assertEqual(deserialize_config('a:b:c'), ['a', 'b', 'c']) self.assertEqual(deserialize_config('a:{"b":1}'), ['a', {'b': 1}]) self.assertEqual(deserialize_config('a:{"b":1'), ['a', '{"b":1']) diff --git a/tests/test_iterators.py b/tests/test_iterators.py index df281c51..f8da2ad1 100755 --- a/tests/test_iterators.py +++ b/tests/test_iterators.py @@ -6,7 +6,7 @@ class TestConventions(unittest.TestCase): def test_flatten(self): - from qatools.iterators import flatten + from qaboard.iterators import flatten self.assertEqual(list(flatten([])), []) self.assertEqual(list(flatten([1])), [1]) self.assertEqual(list(flatten([1, [2]])), [1, 2]) @@ -14,7 +14,7 @@ def test_flatten(self): self.assertEqual(list(flatten([1, {"cde": [2, 3]} ])), [1, {"cde": [2, 3]}]) def test_resolve_aliases(self): - from qatools.iterators import resolve_aliases + from qaboard.iterators import resolve_aliases self.assertEqual( list(resolve_aliases(["a", "b"], {"c": ["d", "e"], "f": ["g", "h"]})), ['a', 'b'] diff --git a/website/docusaurus.config.js b/website/docusaurus.config.js index f25cb360..bda93b6a 100755 --- a/website/docusaurus.config.js +++ b/website/docusaurus.config.js @@ -17,7 +17,6 @@ var config = { tagline: "Visualize and compare algorithm results. Optimize parameters. Share results and track progress.", // You may provide arbitrary config keys to be used as needed by your // template. For example, if you need your repo's URL... - // repoUrl: 'https://gitlab-srv/common-infrastructure/qatools', // scripts: [ // Add custom scripts here that would be placed in