From c344b2246580aedb7f106e0ebca5f4f737c44e93 Mon Sep 17 00:00:00 2001 From: Aleksandra Ovchinnikova Date: Tue, 26 Sep 2023 14:31:34 -0700 Subject: [PATCH] Add docs for streams, fix sync stream and add tests --- README.md | 1 + connect/cli/plugins/commerce/utils.py | 27 ++- docs/stream_usage.md | 73 ++++++++ tests/conftest.py | 5 + tests/plugins/commerce/test_commands.py | 27 +++ tests/plugins/commerce/test_utils.py | 212 +++++++++++++++++++++++- 6 files changed, 323 insertions(+), 22 deletions(-) create mode 100644 docs/stream_usage.md diff --git a/README.md b/README.md index 80d445ea..9df3a5e6 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,7 @@ You can download its zip file from the [Github Releases](https://github.com/clou * [Reports](docs/reports_usage.md) * [Translations](docs/translations_usage.md) * [Projects](docs/project_usage.md) +* [Streams](docs/stream_usage.md) ## Development diff --git a/connect/cli/plugins/commerce/utils.py b/connect/cli/plugins/commerce/utils.py index 8c190bce..88c509c0 100644 --- a/connect/cli/plugins/commerce/utils.py +++ b/connect/cli/plugins/commerce/utils.py @@ -886,7 +886,7 @@ def update_general_information( ) .first() ) - body = {} + body = {'context': {}} updated = 0 errors_on_update = 0 for n in range(2, sheet.max_row + 1): @@ -901,31 +901,24 @@ def update_general_information( h.value == 'Product ID' and stream.get('context', {}).get('product', {}).get('id', None) != v.value ): - if 'context' in body: - body['context'].update({'product': {'id': v.value}}) - else: - body['context'] = {'product': {'id': v.value}} + body['context']['product'] = {'id': v.value} updated += 1 elif ( h.value == 'Partner ID' and stream.get('context', {}).get('account', {}).get('id', None) != v.value ): - if 'context' in body: - body['context'].update({'account': {'id': v.value}}) - else: - body['context'] = {'account': {'id': v.value}} + body['context']['account'] = {'id': v.value} updated += 1 elif ( h.value == 'Marketplace ID' and stream.get('context', {}).get('marketplace', {}).get('id', None) != v.value ): - if 'context' in body: - body['context'].update({'marketplace': {'id': v.value}}) - else: - body['context'] = {'marketplace': {'id': v.value}} + body['context']['marketplace'] = {'id': v.value} updated += 1 if updated: + if not body['context']: + del body['context'] try: client.ns(collection).streams[stream_id].update( json=body, @@ -968,14 +961,14 @@ def update_transformations( try: to_update = {} - if origin_trf['settings'] != settings.value: - to_update['settings'] = settings.value + if origin_trf['settings'] != json.loads(settings.value): + to_update['settings'] = json.loads(settings.value) if descr.value and ( 'description' not in origin_trf or origin_trf['description'] != descr.value ): to_update['description'] = descr.value - if origin_trf['position'] != position.value: - to_update['position'] = position.value + if int(origin_trf['position']) != position.value: + to_update['position'] = int(position.value) if to_update: client.ns(collection).streams[stream_id].transformations[id.value].update( diff --git a/docs/stream_usage.md b/docs/stream_usage.md new file mode 100644 index 00000000..431df4f8 --- /dev/null +++ b/docs/stream_usage.md @@ -0,0 +1,73 @@ +# Streams management + +The streams management area offers commands that allow users to clone and get listing of streams, +export or synchronize to/from Excel file. + +To access the group of commands related to the management of streams you must invoke the CLI with the `commerce stream` command: + +```sh +$ ccli commerce stream + +Usage: ccli commerce [OPTIONS] COMMAND [ARGS]... + +Options: + -h, --help Show this message and exit. + +Commands: + clone Create a clone of a stream. + export Export commerce billing or pricing streams. + list List commerce billing and pricing streams. + sync Synchronize a stream from an excel file. +``` + +### Clone streams + +To clone existing stream you can run: + +```sh +$ccli commerce stream clone [OPTIONS] stream_id +``` + +``` +Options: + -d, --destination_account TEXT Destination account ID + -n, --new-stream-name TEXT Cloned stream name + -v, --validate Executes the validate action after the clone. +``` + +### List streams + +To list available streams you can run: + +```sh +$ ccli commerce stream list [OPTIONS] +``` + +``` +Options: + -q, --query TEXT RQL query expression. +``` + +### Export streams + +To export stream you can run: + +```sh +$ ccli commerce stream export [OPTIONS] stream_id +``` + +``` +Options: + -o, --out FILE Output Excel file name. + -p, --output_path DIRECTORY Directory where to store the export. +``` + +### Synchronize streams + +To synchronize a stream from an Excel file you can run: + +```sh +$ ccli commerce stream sync input_file +``` + +Structure of sync file can be taken from `tests/fixtures/commerce/stream_sync.xlsx` file. diff --git a/tests/conftest.py b/tests/conftest.py index e1c62f8b..a8da7d8d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -336,6 +336,11 @@ def sample_translation_workbook(fs): return load_workbook('./tests/fixtures/translation.xlsx') +@pytest.fixture(scope='function') +def sample_stream_workbook(fs): + return load_workbook('./tests/fixtures/commerce/stream_sync.xlsx') + + @pytest.fixture(scope='function') def mocked_translation_response(): with open('./tests/fixtures/translation_response.json') as response: diff --git a/tests/plugins/commerce/test_commands.py b/tests/plugins/commerce/test_commands.py index 130e1150..a2b77b53 100644 --- a/tests/plugins/commerce/test_commands.py +++ b/tests/plugins/commerce/test_commands.py @@ -942,3 +942,30 @@ def test_sync_stream( assert result.exit_code == 0 assert mocked_cmd_console.echo.call_count == 2 + + +def test_sync_stream_no_stream( + mocker, + ccli, + config_mocker, +): + mocker.patch('connect.cli.plugins.commerce.commands.console') + mocker.patch('connect.cli.plugins.commerce.utils.get_work_book') + mocker.patch( + 'connect.cli.plugins.commerce.utils.guess_if_billing_or_pricing_stream', + return_value=None, + ) + runner = CliRunner() + cmd = [ + 'commerce', + 'stream', + 'sync', + 'STR-7748-7021-7449', + ] + result = runner.invoke( + ccli, + cmd, + ) + + assert result.exit_code == 1 + assert 'Stream STR-7748-7021-7449 not found for the current account VA-000.' in result.output diff --git a/tests/plugins/commerce/test_utils.py b/tests/plugins/commerce/test_utils.py index 37daccee..782f6f01 100644 --- a/tests/plugins/commerce/test_utils.py +++ b/tests/plugins/commerce/test_utils.py @@ -1,17 +1,22 @@ +import json import os import tempfile -import json import pytest from click import ClickException -from openpyxl import Workbook +from connect.client import ConnectClient +from openpyxl import Workbook, load_workbook from connect.cli.plugins.commerce.utils import ( - fill_and_download_attachments, - guess_if_billing_or_pricing_stream, + _validate_header, clone_stream, - create_stream_from_origin, clone_transformations, + create_stream_from_origin, + fill_and_download_attachments, + get_work_book, + guess_if_billing_or_pricing_stream, + update_general_information, + update_transformations, ) @@ -279,3 +284,200 @@ def test_clone_transformations_fix_file_mapping( mocked_col_mapping.assert_called() mocked_progress.add_task.assert_called_with('Processing transformations', total=1) assert mocked_progress.update.call_count == 1 + + +def test_get_work_book(): + wb = get_work_book('./tests/fixtures/commerce/stream_sync.xlsx') + + general_info = wb['General Information'] + assert general_info['B2'].value == 'STR-7748-7021-7449' + + wb.close() + + +def test_get_work_book_invalid_sheets(): + with pytest.raises(ClickException) as ce: + get_work_book('./tests/fixtures/actions_sync.xlsx') + + assert str(ce.value) == ( + 'The file must contain `General Information`, `Columns`, `Transformations` and ' + '`Attachments` sheets.' + ) + + +def test_get_work_book_non_existing_file(): + with pytest.raises(ClickException) as ce: + get_work_book('non_existing_file.xlsx') + + assert str(ce.value) == 'The file non_existing_file.xlsx does not exists.' + + +def test_get_work_book_invalid_format(): + with pytest.raises(ClickException) as ce: + get_work_book('./tests/fixtures/image.png') + + assert str(ce.value) == 'The file ./tests/fixtures/image.png has invalid format, must be xlsx.' + + +def test_validate_header_missing(): + with pytest.raises(ClickException) as ce: + _validate_header(['ID', 'Name'], ['ID', 'Name', 'Description'], 'Streams') + + assert str(ce.value) == 'The Streams sheet header does not contain `Description` header.' + + +def test_update_general_information_error(sample_stream_workbook, mocker, mocked_responses): + client = ConnectClient( + api_key='ApiKey X', + endpoint='https://localhost/public/v1', + use_specs=False, + ) + mocked_responses.add( + method='GET', + url='https://localhost/public/v1/billing/streams?' + 'eq(id,STR-7755-7115-2464)&select(context,samples,sources)&limit=1&offset=0', + json=[{'name': 'Name', 'description': 'Description'}], + ) + + mocked_responses.add( + method='PUT', + url='https://localhost/public/v1/billing/streams/STR-7755-7115-2464', + status=400, + ) + + results = [] + update_general_information( + client=client, + collection='billing', + stream_id='STR-7755-7115-2464', + sheet=sample_stream_workbook['General Information'], + results=results, + errors=[], + progress=mocker.MagicMock(), + ) + + sample_stream_workbook.close() + assert len(results) == 1 + assert results[0] == ('General information', 5, 0, 0, 0, 0, 5) + + +def test_update_transformations(sample_stream_workbook, mocker, mocked_responses): + client = ConnectClient( + api_key='ApiKey X', + endpoint='https://localhost/public/v1', + use_specs=False, + ) + + mocked_responses.add( + method='GET', + url='https://localhost/public/v1/billing/streams/STR-7755-7115-2464/transformations', + json=[ + {'id': 'STRA-774-870-217-449-001'}, + {'id': 'STRA-774-870-217-449-002'}, + ], + ) + mocked_responses.add( + method='GET', + url='https://localhost/public/v1/billing/streams/STR-7755-7115-2464' + '/transformations/STRA-774-870-217-449-001', + json={ + 'id': 'STRA-774-870-217-449-001', + 'settings': { + 'from': 'id', + 'regex': { + 'groups': {'1': {'name': 'first_name', 'type': 'string'}}, + 'pattern': '(?P\\w+)', + }, + }, + 'description': 'Old description', + 'position': 10000, + }, + ) + mocked_responses.add( + method='GET', + url='https://localhost/public/v1/billing/streams/STR-7755-7115-2464' + '/transformations/STRA-774-870-217-449-002', + json={ + 'id': 'STRA-774-870-217-449-002', + 'settings': { + 'additional_values': [], + 'from': 'position', + 'match_condition': True, + 'value': '200', + }, + 'description': 'edeeasd', + 'position': 20000, + }, + ) + mocked_responses.add( + method='PUT', + url='https://localhost/public/v1/billing/streams/STR-7755-7115-2464' + '/transformations/STRA-774-870-217-449-001', + status=200, + ) + + results = [] + errors = [] + update_transformations( + client=client, + collection='billing', + stream_id='STR-7755-7115-2464', + sheet=sample_stream_workbook['Transformations'], + results=results, + errors=errors, + progress=mocker.MagicMock(), + ) + + assert len(errors) == 0 + assert len(results) == 1 + assert results[0] == ('Transformations', 2, 0, 1, 0, 0, 0) + + +def test_update_transformations_not_exists(sample_stream_workbook, mocker, mocked_responses): + client = ConnectClient( + api_key='ApiKey X', + endpoint='https://localhost/public/v1', + use_specs=False, + ) + + mocked_responses.add( + method='GET', + url='https://localhost/public/v1/billing/streams/STR-7755-7115-2464' + '/transformations/STRA-774-870-217-449-001', + status=404, + ) + mocked_responses.add( + method='GET', + url='https://localhost/public/v1/billing/streams/STR-7755-7115-2464' + '/transformations/STRA-774-870-217-449-002', + status=404, + ) + mocked_responses.add( + method='GET', + url='https://localhost/public/v1/billing/streams/STR-7755-7115-2464/transformations', + json=[], + ) + + results = [] + errors = [] + update_transformations( + client=client, + collection='billing', + stream_id='STR-7755-7115-2464', + sheet=sample_stream_workbook['Transformations'], + results=results, + errors=errors, + progress=mocker.MagicMock(), + ) + + assert len(errors) == 2 + assert ( + 'The transformation STRA-774-870-217-449-001' + ' cannot be updated because it does not exist.' + ) in errors + assert ( + 'The transformation STRA-774-870-217-449-002' + ' cannot be updated because it does not exist.' + ) in errors + + assert results[0] == ('Transformations', 2, 0, 0, 0, 0, 2)