diff --git a/src/rpft/parsers/creation/contentindexparser.py b/src/rpft/parsers/creation/contentindexparser.py index b1d04ea..2ec9625 100644 --- a/src/rpft/parsers/creation/contentindexparser.py +++ b/src/rpft/parsers/creation/contentindexparser.py @@ -1,5 +1,6 @@ import importlib from collections import OrderedDict +from typing import Dict, List from rpft.logger.logger import get_logger, logging_context from rpft.parsers.common.cellparser import CellParser @@ -35,10 +36,10 @@ def __init__(self, rows, row_model): self.row_model = row_model def to_dict(self): - rows = [] - for content in self.rows.values(): - rows.append(content.dict()) - return {"model": self.row_model.__name__, "rows": rows} + return { + "model": self.row_model.__name__, + "rows": [content.dict() for content in self.rows.values()], + } class ParserError(Exception): @@ -56,16 +57,14 @@ def __init__( self.tag_matcher = tag_matcher self.template_sheets = {} self.data_sheets = {} - self.flow_definition_rows = [] # list of ContentIndexRowModel - self.campaign_parsers = {} # name-indexed dict of CampaignParser - self.trigger_parsers = [] - - self.user_models_module = None - if user_data_model_module_name: - self.user_models_module = importlib.import_module( - user_data_model_module_name - ) - + self.flow_definition_rows: List[ContentIndexRowModel] = [] + self.campaign_parsers: Dict[str, tuple[str, CampaignParser]] = {} + self.trigger_parsers = OrderedDict() + self.user_models_module = ( + importlib.import_module(user_data_model_module_name) + if user_data_model_module_name + else None + ) indices = self.reader.get_sheets_by_name("content_index") if not indices: @@ -77,55 +76,66 @@ def __init__( self._populate_missing_templates() def _process_content_index_table(self, sheet: Sheet): - row_parser = RowParser(ContentIndexRowModel, CellParser()) - sheet_parser = SheetParser(row_parser, sheet.table) - content_index_rows = sheet_parser.parse_all() - for row_idx, row in enumerate(content_index_rows, start=2): + sheet_parser = SheetParser( + RowParser(ContentIndexRowModel, CellParser()), + sheet.table, + ) + + for row_idx, row in enumerate(sheet_parser.parse_all(), start=2): logging_prefix = f"{sheet.reader.name}-{sheet.name} | row {row_idx}" + with logging_context(logging_prefix): if row.status == "draft": continue + if not self.tag_matcher.matches(row.tags): continue + if len(row.sheet_name) != 1 and row.type != "data_sheet": LOGGER.critical( - f"For {row.type} rows, " - "exactly one sheet_name has to be specified" + f"For {row.type} rows, exactly one sheet_name has to be" + " specified" ) + if row.type == "content_index": - sheet_name = row.sheet_name[0] - sheet = self._get_sheet_or_die(sheet_name) + sheet = self._get_sheet_or_die(row.sheet_name[0]) + with logging_context(f"{sheet.name}"): self._process_content_index_table(sheet) elif row.type == "data_sheet": if not len(row.sheet_name) >= 1: LOGGER.critical( - "For data_sheet rows, at least one " - "sheet_name has to be specified" + "For data_sheet rows, at least one sheet_name has to be" + " specified" ) + self._process_data_sheet(row) - elif row.type in ["template_definition", "create_flow"]: - if row.type == "template_definition": - if row.new_name: - LOGGER.warning( - "template_definition does not support 'new_name'; " - f"new_name '{row.new_name}' will be ignored." - ) - self._add_template(row, True) - else: - self.flow_definition_rows.append((logging_prefix, row)) + elif row.type == "template_definition": + if row.new_name: + LOGGER.warning( + "template_definition does not support 'new_name'; " + f"new_name '{row.new_name}' will be ignored." + ) + + self._add_template(row, True) + elif row.type == "create_flow": + self.flow_definition_rows.append((logging_prefix, row)) elif row.type == "create_campaign": campaign_parser = self.create_campaign_parser(row) name = campaign_parser.campaign.name + if name in self.campaign_parsers: LOGGER.warning( f"Duplicate campaign definition sheet '{name}'. " "Overwriting previous definition." ) + self.campaign_parsers[name] = (logging_prefix, campaign_parser) elif row.type == "create_triggers": - trigger_parser = self.create_trigger_parser(row) - self.trigger_parsers.append((logging_prefix, trigger_parser)) + self.trigger_parsers[row.sheet_name[0]] = ( + logging_prefix, + self.create_trigger_parser(row), + ) elif row.type == "ignore_row": self._process_ignore_row(row.sheet_name[0]) else: @@ -139,6 +149,7 @@ def _add_template(self, row, update_duplicates=False): f"Duplicate template definition sheet '{sheet_name}'. " "Overwriting previous definition." ) + if sheet_name not in self.template_sheets or update_duplicates: sheet = self._get_sheet_or_die(sheet_name) self.template_sheets[sheet_name] = TemplateSheet( @@ -146,21 +157,13 @@ def _add_template(self, row, update_duplicates=False): ) def _process_ignore_row(self, sheet_name): - # Remove the flow definition row for the given name self.flow_definition_rows = [ (logging_prefix, row) for logging_prefix, row in self.flow_definition_rows if (row.new_name or row.sheet_name[0]) != sheet_name ] - # Template definitions are NOT removed, as their existence - # has no effect on the output flows. - # Remove campaign/trigger definitions with the given name self.campaign_parsers.pop(sheet_name, None) - self.trigger_parsers = [ - (logging_prefix, trigger_parser) - for logging_prefix, trigger_parser in self.trigger_parsers - if trigger_parser.sheet_name != sheet_name - ] + self.trigger_parsers.pop(sheet_name, None) def _populate_missing_templates(self): for logging_prefix, row in self.flow_definition_rows: @@ -171,10 +174,7 @@ def _get_sheet_or_die(self, sheet_name): candidates = self.reader.get_sheets_by_name(sheet_name) if not candidates: - raise ParserError( - "Sheet not found", - {"name": sheet_name}, - ) + raise ParserError("Sheet not found", {"name": sheet_name}) active = candidates[-1] @@ -195,11 +195,13 @@ def _get_sheet_or_die(self, sheet_name): def _process_data_sheet(self, row): sheet_names = row.sheet_name + if row.operation.type in ["filter", "sort"] and len(sheet_names) > 1: LOGGER.warning( "data_sheet definition take only one sheet_name for filter and sort " "operations. All but the first sheet_name are ignored." ) + if not row.operation.type: if len(sheet_names) > 1: LOGGER.warning( @@ -208,12 +210,14 @@ def _process_data_sheet(self, row): "in the future." ) data_sheet = self._data_sheets_concat(sheet_names, row.data_model) + else: if not row.new_name: LOGGER.critical( - "If an operation is applied to a data_sheet, " - "a new_name has to be provided" + "If an operation is applied to a data_sheet, a new_name has to be" + " provided" ) + if row.operation.type == "concat": data_sheet = self._data_sheets_concat(sheet_names, row.data_model) elif row.operation.type == "filter": @@ -226,11 +230,14 @@ def _process_data_sheet(self, row): ) else: LOGGER.critical(f'Unknown operation "{row.operation}"') + new_name = row.new_name or sheet_names[0] + if new_name in self.data_sheets: LOGGER.warn( f"Duplicate data sheet {new_name}. Overwriting previous definition." ) + self.data_sheets[new_name] = data_sheet def _get_data_sheet(self, sheet_name, data_model_name): @@ -241,6 +248,7 @@ def _get_data_sheet(self, sheet_name, data_model_name): def _get_new_data_sheet(self, sheet_name, data_model_name=None): user_model = None + if self.user_models_module and data_model_name: try: user_model = getattr(self.user_models_module, data_model_name) @@ -249,40 +257,51 @@ def _get_new_data_sheet(self, sheet_name, data_model_name=None): f'Undefined data_model_name "{data_model_name}" ' f"in {self.user_models_module}." ) + data_table = self._get_sheet_or_die(sheet_name) + with logging_context(sheet_name): data_table = self._get_sheet_or_die(sheet_name).table + if not user_model: LOGGER.info("Inferring RowModel automatically") user_model = model_from_headers(sheet_name, data_table.headers) - row_parser = RowParser(user_model, CellParser()) - sheet_parser = SheetParser(row_parser, data_table) - data_rows = sheet_parser.parse_all() + + data_rows = SheetParser( + RowParser(user_model, CellParser()), + data_table, + ).parse_all() model_instances = OrderedDict((row.ID, row) for row in data_rows) + return DataSheet(model_instances, user_model) def _data_sheets_concat(self, sheet_names, data_model_name): all_data_rows = OrderedDict() user_model = None + for sheet_name in sheet_names: with logging_context(sheet_name): data_sheet = self._get_data_sheet(sheet_name, data_model_name) + if user_model and user_model is not data_sheet.row_model: LOGGER.critical( - "Cannot concatenate data_sheets with different " - "underlying models" + "Cannot concatenate data_sheets with different underlying" + " models" ) + user_model = data_sheet.row_model all_data_rows.update(data_sheet.rows) + return DataSheet(all_data_rows, user_model) def _data_sheets_filter(self, sheet_name, data_model_name, operation): data_sheet = self._get_data_sheet(sheet_name, data_model_name) new_row_data = OrderedDict() - for rowID, row in data_sheet.rows.items(): + + for row_id, row in data_sheet.rows.items(): try: if eval(operation.expression, {}, dict(row)) is True: - new_row_data[rowID] = row + new_row_data[row_id] = row except NameError as e: LOGGER.critical(f"Invalid filtering expression: {e}") except SyntaxError as e: @@ -290,16 +309,19 @@ def _data_sheets_filter(self, sheet_name, data_model_name, operation): f'Invalid filtering expression: "{e.text}". ' f"SyntaxError at line {e.lineno} character {e.offset}" ) + return DataSheet(new_row_data, data_sheet.row_model) def _data_sheets_sort(self, sheet_name, data_model_name, operation): data_sheet = self._get_data_sheet(sheet_name, data_model_name) - reverse = True if operation.order.lower() == "descending" else False + try: - new_row_data_list = sorted( - data_sheet.rows.items(), - key=lambda kvpair: eval(operation.expression, {}, dict(kvpair[1])), - reverse=reverse, + new_row_data = OrderedDict( + sorted( + data_sheet.rows.items(), + key=lambda kvpair: eval(operation.expression, {}, dict(kvpair[1])), + reverse=operation.order.lower() == "descending", + ) ) except NameError as e: LOGGER.critical(f"Invalid sorting expression: {e}") @@ -309,9 +331,6 @@ def _data_sheets_sort(self, sheet_name, data_model_name, operation): f"SyntaxError at line {e.lineno} character {e.offset}" ) - new_row_data = OrderedDict() - for k, v in new_row_data_list: - new_row_data[k] = v return DataSheet(new_row_data, data_sheet.row_model) def get_data_sheet_row(self, sheet_name, row_id): @@ -338,14 +357,16 @@ def get_node_group( ) else: LOGGER.critical( - "For insert_as_block, either both data_sheet and data_row_id " - "or neither have to be provided." + "For insert_as_block, either both data_sheet and data_row_id or neither" + " have to be provided." ) def data_sheets_to_dict(self): sheets = {} + for sheet_name, sheet in self.data_sheets.items(): sheets[sheet_name] = sheet.to_dict() + return { "sheets": sheets, "meta": { @@ -359,45 +380,55 @@ def parse_all(self): self.parse_all_flows(rapidpro_container) self.parse_all_campaigns(rapidpro_container) self.parse_all_triggers(rapidpro_container) + return rapidpro_container def create_campaign_parser(self, row): sheet_name = row.sheet_name[0] sheet = self._get_sheet_or_die(sheet_name) - row_parser = RowParser(CampaignEventRowModel, CellParser()) - sheet_parser = SheetParser(row_parser, sheet.table) - rows = sheet_parser.parse_all() + rows = SheetParser( + RowParser(CampaignEventRowModel, CellParser()), + sheet.table, + ).parse_all() + return CampaignParser(row.new_name or sheet_name, row.group, rows) def create_trigger_parser(self, row): sheet_name = row.sheet_name[0] sheet = self._get_sheet_or_die(sheet_name) - row_parser = RowParser(TriggerRowModel, CellParser()) - sheet_parser = SheetParser(row_parser, sheet.table) - rows = sheet_parser.parse_all() + rows = SheetParser( + RowParser(TriggerRowModel, CellParser()), + sheet.table, + ).parse_all() + return TriggerParser(sheet_name, rows) def parse_all_campaigns(self, rapidpro_container): for logging_prefix, campaign_parser in self.campaign_parsers.values(): sheet_name = campaign_parser.campaign.name + with logging_context(f"{logging_prefix} | {sheet_name}"): campaign = campaign_parser.parse() rapidpro_container.add_campaign(campaign) def parse_all_triggers(self, rapidpro_container): - for logging_prefix, trigger_parser in self.trigger_parsers: + for logging_prefix, trigger_parser in self.trigger_parsers.values(): sheet_name = trigger_parser.sheet_name + with logging_context(f"{logging_prefix} | {sheet_name}"): triggers = trigger_parser.parse() + for trigger in triggers: rapidpro_container.add_trigger(trigger) def parse_all_flows(self, rapidpro_container): flows = {} + for logging_prefix, row in self.flow_definition_rows: with logging_context(f"{logging_prefix} | {row.sheet_name[0]}"): if row.data_sheet and not row.data_row_id: data_rows = self.get_data_sheet_rows(row.data_sheet) + for data_row_id in data_rows.keys(): with logging_context(f'with data_row_id "{data_row_id}"'): flow = self._parse_flow( @@ -408,16 +439,18 @@ def parse_all_flows(self, rapidpro_container): rapidpro_container, row.new_name, ) + if flow.name in flows: LOGGER.warning( f"Multiple definitions of flow '{flow.name}'. " "Overwriting." ) + flows[flow.name] = flow elif not row.data_sheet and row.data_row_id: LOGGER.critical( - "For create_flow, if data_row_id is provided, " - "data_sheet must also be provided." + "For create_flow, if data_row_id is provided, data_sheet must" + " also be provided." ) else: flow = self._parse_flow( @@ -428,12 +461,15 @@ def parse_all_flows(self, rapidpro_container): rapidpro_container, row.new_name, ) + if flow.name in flows: LOGGER.warning( f"Multiple definitions of flow '{flow.name}'. " "Overwriting." ) + flows[flow.name] = flow + for flow in flows.values(): rapidpro_container.add_flow(flow) @@ -448,6 +484,7 @@ def _parse_flow( parse_as_block=False, ): base_name = new_name or sheet_name + if data_sheet and data_row_id: flow_name = " - ".join([base_name, data_row_id]) context = self.get_data_sheet_row(data_sheet, data_row_id) @@ -457,55 +494,66 @@ def _parse_flow( "For create_flow, if no data_sheet is provided, " "data_row_id should be blank as well." ) + flow_name = base_name context = {} + template_sheet = self.get_template_sheet(sheet_name) - template_table = template_sheet.table - template_argument_definitions = template_sheet.argument_definitions - context = dict(context) - self.map_template_arguments_to_context( - template_argument_definitions, template_arguments, context + context = self.map_template_arguments_to_context( + template_sheet.argument_definitions, + template_arguments, + dict(context), ) flow_parser = FlowParser( rapidpro_container, flow_name, - template_table, + template_sheet.table, context=context, content_index_parser=self, ) + if parse_as_block: return flow_parser.parse_as_block() else: return flow_parser.parse(add_to_container=False) - def map_template_arguments_to_context(self, arg_defs, args, context): - # Template arguments are positional arguments. - # This function maps them to the arguments from the template - # definition, and adds the values of the arguments to the context - # with the appropriate variable name (from the definition) + def map_template_arguments_to_context(self, arg_defs, args, context) -> dict: + """ + Map template arguments, which are positional, to the arguments from the template + definition, and add the values of the arguments to the context with the + appropriate variable name (from the definition). + """ if len(args) > len(arg_defs): - # Check if these args are non-empty. - # Once the row parser is cleaned up to eliminate trailing '' - # entries, this won't be necessary + # Once the row parser is cleaned up to eliminate trailing '' entries, this + # won't be necessary extra_args = args[len(arg_defs) :] non_empty_extra_args = [ea for ea in extra_args if ea] + if non_empty_extra_args: LOGGER.warn("Too many arguments provided to template") - # All extra args are blank. Truncate them + args = args[: len(arg_defs)] + args_padding = [""] * (len(arg_defs) - len(args)) + for arg_def, arg in zip(arg_defs, args + args_padding): if arg_def.name in context: LOGGER.critical( f'Template argument "{arg_def.name}" doubly defined ' f'in context: "{context}"' ) + arg_value = arg if arg != "" else arg_def.default_value + if arg_value == "": LOGGER.critical( f'Required template argument "{arg_def.name}" not provided' ) - if arg_def.type == "sheet": - context[arg_def.name] = self.get_data_sheet_rows(arg_value) - else: - context[arg_def.name] = arg_value + + context[arg_def.name] = ( + self.get_data_sheet_rows(arg_value) + if arg_def.type == "sheet" + else arg_value + ) + + return context diff --git a/tests/test_contentindexparser.py b/tests/test_contentindexparser.py index 506203b..a19cc2c 100644 --- a/tests/test_contentindexparser.py +++ b/tests/test_contentindexparser.py @@ -1,4 +1,5 @@ -import unittest +from typing import Set +from unittest import TestCase from rpft.parsers.creation.contentindexparser import ContentIndexParser from rpft.parsers.creation.tagmatcher import TagMatcher @@ -13,24 +14,31 @@ def csv_join(*args): return "\n".join(args) + "\n" -class TestTemplate(unittest.TestCase): - def compare_messages(self, render_output, flow_name, messages_exp, context=None): - flow_found = False - for flow in render_output["flows"]: - if flow["name"] == flow_name: - flow_found = True - actions = traverse_flow(flow, context or Context()) - actions_exp = list(zip(["send_msg"] * len(messages_exp), messages_exp)) - self.assertEqual(actions, actions_exp) - if not flow_found: - self.assertTrue( - False, msg=f'Flow with name "{flow_name}" does not exist in output.' - ) +class TestTemplate(TestCase): + def assertFlowMessages(self, render_output, flow_name, messages_exp, context=None): + flows = [flow for flow in render_output["flows"] if flow["name"] == flow_name] + self.assertTrue( + len(flows) > 0, + msg=f'Flow with name "{flow_name}" does not exist in output.', + ) -class TestParsing(TestTemplate): - def get_flow_names(self, render_output): - return {flow["name"] for flow in render_output["flows"]} + actions = traverse_flow(flows[0], context or Context()) + actions_exp = list(zip(["send_msg"] * len(messages_exp), messages_exp)) + + self.assertEqual(actions, actions_exp) + + +class TestTemplateDefinition(TestCase): + def setUp(self): + template = csv_join( + "row_id,type,from,message_text", + ",send_message,start,Some text", + ) + self.sheet_data_dict = { + "my_template": template, + "my_template2": template, + } def test_basic_template_definition(self): ci_sheet = ( @@ -38,34 +46,43 @@ def test_basic_template_definition(self): "template_definition,my_template,,,,,\n" "template_definition,my_template2,,,,,draft\n" ) - self.check_basic_template_definition(ci_sheet) + self.parser = ContentIndexParser( + MockSheetReader( + ci_sheet, + self.sheet_data_dict, + ) + ) + + self.assertTemplateDefinition() def test_ignore_template_definition(self): - # Ensure that ignoring a template row does NOT remove the template + """ + Ensure that ignoring a template row does NOT remove the template + """ ci_sheet = ( "type,sheet_name,data_sheet,data_row_id,new_name,data_model,status\n" "template_definition,my_template,,,,,\n" "ignore_row,my_template,,,,,\n" ) - self.check_basic_template_definition(ci_sheet) - - def check_basic_template_definition(self, ci_sheet): - my_template = csv_join( - "row_id,type,from,message_text", - ",send_message,start,Some text", + self.parser = ContentIndexParser( + MockSheetReader( + ci_sheet, + self.sheet_data_dict, + ) ) - sheet_reader = MockSheetReader( - ci_sheet, - {"my_template": my_template, "my_template2": my_template}, - ) - ci_parser = ContentIndexParser(sheet_reader) - template_sheet = ci_parser.get_template_sheet("my_template") + self.assertTemplateDefinition() + + def assertTemplateDefinition(self): + template_sheet = self.parser.get_template_sheet("my_template") + self.assertEqual(template_sheet.table[0][1], "send_message") self.assertEqual(template_sheet.table[0][3], "Some text") with self.assertRaises(KeyError): - ci_parser.get_template_sheet("my_template2") + self.parser.get_template_sheet("my_template2") + +class TestParsing(TestTemplate): def test_basic_nesting(self): ci_sheet = ( "type,sheet_name,data_sheet,data_row_id,new_name,data_model,status\n" @@ -89,13 +106,16 @@ def test_basic_nesting(self): "my_template": my_template, "my_template2": my_template2, } + ci_parser = ContentIndexParser(MockSheetReader(ci_sheet, sheet_dict)) - sheet_reader = MockSheetReader(ci_sheet, sheet_dict) - ci_parser = ContentIndexParser(sheet_reader) - template_sheet = ci_parser.get_template_sheet("my_template") - self.assertEqual(template_sheet.table[0][3], "Some text") - template_sheet = ci_parser.get_template_sheet("my_template2") - self.assertEqual(template_sheet.table[0][3], "Other text") + self.assertEqual( + ci_parser.get_template_sheet("my_template").table[0][3], + "Some text", + ) + self.assertEqual( + ci_parser.get_template_sheet("my_template2").table[0][3], + "Other text", + ) def test_basic_user_model(self): ci_sheet = ( @@ -107,11 +127,13 @@ def test_basic_user_model(self): "rowA,1A,2A", "rowB,1B,2B", ) - - sheet_reader = MockSheetReader(ci_sheet, {"simpledata": simpledata}) - ci_parser = ContentIndexParser(sheet_reader, "tests.datarowmodels.simplemodel") + ci_parser = ContentIndexParser( + MockSheetReader(ci_sheet, {"simpledata": simpledata}), + "tests.datarowmodels.simplemodel", + ) datamodelA = ci_parser.get_data_sheet_row("simpledata", "rowA") datamodelB = ci_parser.get_data_sheet_row("simpledata", "rowB") + self.assertEqual(datamodelA.value1, "1A") self.assertEqual(datamodelA.value2, "2A") self.assertEqual(datamodelB.value1, "1B") @@ -133,11 +155,15 @@ def test_ignore_flow_definition(self): sheet_dict = { "my_basic_flow": my_basic_flow, } + render_output = ( + ContentIndexParser( + MockSheetReader(ci_sheet, sheet_dict), + "tests.datarowmodels.nestedmodel", + ) + .parse_all() + .render() + ) - sheet_reader = MockSheetReader(ci_sheet, sheet_dict) - ci_parser = ContentIndexParser(sheet_reader, "tests.datarowmodels.nestedmodel") - container = ci_parser.parse_all() - render_output = container.render() self.assertEqual(len(render_output["flows"]), 1) self.assertEqual(render_output["flows"][0]["name"], "my_renamed_basic_flow2") @@ -167,15 +193,18 @@ def test_ignore_templated_flow_definition(self): "nesteddata": nesteddata, "my_template": my_template, } + render_output = ( + ContentIndexParser( + MockSheetReader(ci_sheet, sheet_dict), + "tests.datarowmodels.nestedmodel", + ) + .parse_all() + .render() + ) - sheet_reader = MockSheetReader(ci_sheet, sheet_dict) - ci_parser = ContentIndexParser(sheet_reader, "tests.datarowmodels.nestedmodel") - container = ci_parser.parse_all() - render_output = container.render() self.assertEqual(len(render_output["flows"]), 3) - names = {flow["name"] for flow in render_output["flows"]} - self.assertEqual( - names, + self.assertFlowNamesEqual( + render_output, { "bulk_renamed - row1", "bulk_renamed - row2", @@ -191,7 +220,6 @@ def test_generate_flows(self): "create_flow,my_basic_flow,,,,,\n" "data_sheet,nesteddata,,,,NestedRowModel,\n" ) - # The templates are instantiated implicitly with all data rows ci_sheet_alt = ( "type,sheet_name,data_sheet,data_row_id,new_name,data_model,status\n" "create_flow,my_template,nesteddata,,,,\n" @@ -218,32 +246,58 @@ def test_generate_flows(self): "my_template": my_template, "my_basic_flow": my_basic_flow, } + render_output = ( + ContentIndexParser( + MockSheetReader(ci_sheet, sheet_dict), "tests.datarowmodels.nestedmodel" + ) + .parse_all() + .render() + ) - sheet_reader = MockSheetReader(ci_sheet, sheet_dict) - ci_parser = ContentIndexParser(sheet_reader, "tests.datarowmodels.nestedmodel") - container = ci_parser.parse_all() - render_output = container.render() - self.compare_messages(render_output, "my_basic_flow", ["Some text"]) - self.compare_messages( - render_output, "my_template - row1", ["Value1", "Happy1 and Sad1"] + self.assertFlowMessages( + render_output, + "my_basic_flow", + ["Some text"], + ) + self.assertFlowMessages( + render_output, + "my_template - row1", + ["Value1", "Happy1 and Sad1"], ) - self.compare_messages( - render_output, "my_template - row2", ["Value2", "Happy2 and Sad2"] + self.assertFlowMessages( + render_output, + "my_template - row2", + ["Value2", "Happy2 and Sad2"], ) - sheet_reader = MockSheetReader(ci_sheet_alt, sheet_dict) - ci_parser = ContentIndexParser(sheet_reader, "tests.datarowmodels.nestedmodel") - container = ci_parser.parse_all() - render_output = container.render() - self.compare_messages(render_output, "my_basic_flow", ["Some text"]) - self.compare_messages( - render_output, "my_template - row1", ["Value1", "Happy1 and Sad1"] + render_output = ( + ContentIndexParser( + MockSheetReader(ci_sheet_alt, sheet_dict), + "tests.datarowmodels.nestedmodel", + ) + .parse_all() + .render() ) - self.compare_messages( - render_output, "my_template - row2", ["Value2", "Happy2 and Sad2"] + + self.assertFlowMessages( + render_output, + "my_basic_flow", + ["Some text"], ) - self.compare_messages( - render_output, "my_template - row3", ["Value3", "Happy3 and Sad3"] + self.assertFlowMessages( + render_output, + "my_template - row1", + ["Value1", "Happy1 and Sad1"], + ) + self.assertFlowMessages( + render_output, + "my_template - row2", + ["Value2", "Happy2 and Sad2"], + ) + self.assertFlowMessages( + render_output, + "my_template - row3", + ["Value3", "Happy3 and Sad3"], ) def test_duplicate_create_flow(self): @@ -265,12 +319,13 @@ def test_duplicate_create_flow(self): "my_template": my_template, "my_template2": my_template2, } + render_output = ( + ContentIndexParser(MockSheetReader(ci_sheet, sheet_dict)) + .parse_all() + .render() + ) - sheet_reader = MockSheetReader(ci_sheet, sheet_dict) - ci_parser = ContentIndexParser(sheet_reader) - container = ci_parser.parse_all() - render_output = container.render() - self.compare_messages(render_output, "my_template", ["Other text"]) + self.assertFlowMessages(render_output, "my_template", ["Other text"]) def test_bulk_flows_with_args(self): ci_sheet = ( @@ -293,17 +348,21 @@ def test_bulk_flows_with_args(self): "nesteddata": nesteddata, "my_template": my_template, } + render_output = ( + ContentIndexParser( + MockSheetReader(ci_sheet, sheet_dict), + "tests.datarowmodels.nestedmodel", + ) + .parse_all() + .render() + ) - sheet_reader = MockSheetReader(ci_sheet, sheet_dict) - ci_parser = ContentIndexParser(sheet_reader, "tests.datarowmodels.nestedmodel") - container = ci_parser.parse_all() - render_output = container.render() - self.compare_messages( + self.assertFlowMessages( render_output, "my_renamed_template - row1", ["Value1 ARG1 ARG2", "Happy1 and Sad1"], ) - self.compare_messages( + self.assertFlowMessages( render_output, "my_renamed_template - row2", ["Value2 ARG1 ARG2", "Happy2 and Sad2"], @@ -344,11 +403,6 @@ def test_insert_as_block(self): "my_template": my_template, "my_basic_flow": my_basic_flow, } - - sheet_reader = MockSheetReader(ci_sheet, sheet_dict) - ci_parser = ContentIndexParser(sheet_reader, "tests.datarowmodels.nestedmodel") - container = ci_parser.parse_all() - render_output = container.render() messages_exp = [ "Some text", "Value1", @@ -360,7 +414,16 @@ def test_insert_as_block(self): "Value1", "I'm Sad1", # we're taking the hard exit now, leaving the flow. ] - self.compare_messages( + render_output = ( + ContentIndexParser( + MockSheetReader(ci_sheet, sheet_dict), + "tests.datarowmodels.nestedmodel", + ) + .parse_all() + .render() + ) + + self.assertFlowMessages( render_output, "my_renamed_basic_flow", messages_exp, @@ -408,55 +471,52 @@ def test_insert_as_block_with_sheet_arguments(self): "my_basic_flow": my_basic_flow, "string_lookup": string_lookup, } + render_output = ( + ContentIndexParser( + MockSheetReader(ci_sheet, sheet_dict), + "tests.datarowmodels.listmodel", + ) + .parse_all() + .render() + ) - sheet_reader = MockSheetReader(ci_sheet, sheet_dict) - ci_parser = ContentIndexParser(sheet_reader, "tests.datarowmodels.listmodel") - container = ci_parser.parse_all() - render_output = container.render() - messages_exp = [ - "Some text", - "Hello :)Nice to see you :)", - "Intermission", - "Nice to see you :)Bye :)", - ] - self.compare_messages( + self.assertFlowMessages( render_output, "my_basic_flow", - messages_exp, + [ + "Some text", + "Hello :)Nice to see you :)", + "Intermission", + "Nice to see you :)Bye :)", + ], Context(variables={"@field.mood": "happy"}), ) - messages_exp = [ - "Some text", - "Hello :(Not nice to see you :(", - "Intermission", - "Not nice to see you :(Bye :(", - ] - self.compare_messages( + self.assertFlowMessages( render_output, "my_basic_flow", - messages_exp, + [ + "Some text", + "Hello :(Not nice to see you :(", + "Intermission", + "Not nice to see you :(Bye :(", + ], Context(variables={"@field.mood": "sad"}), ) - messages_exp = [ - "Some text", - "HelloNice to see you", - "Intermission", - "Nice to see youBye", - ] - self.compare_messages( + self.assertFlowMessages( render_output, "my_basic_flow", - messages_exp, + [ + "Some text", + "HelloNice to see you", + "Intermission", + "Nice to see youBye", + ], Context(variables={"@field.mood": "something else"}), ) - - messages_exp = [ - "Hello :)Bye :)", - ] - self.compare_messages( + self.assertFlowMessages( render_output, "my_template - row3", - messages_exp, + ["Hello :)Bye :)"], Context(variables={"@field.mood": "happy"}), ) @@ -474,19 +534,25 @@ def test_insert_as_block_with_arguments(self): sheet_dict = { "my_template": my_template, } + render_output = ( + ContentIndexParser( + MockSheetReader(ci_sheet, sheet_dict), + "tests.datarowmodels.listmodel", + ) + .parse_all() + .render() + ) - sheet_reader = MockSheetReader(ci_sheet, sheet_dict) - ci_parser = ContentIndexParser(sheet_reader, "tests.datarowmodels.listmodel") - container = ci_parser.parse_all() - render_output = container.render() - messages_exp = [ - "value1 default2", - ] - self.compare_messages(render_output, "my_template_default", messages_exp) - messages_exp = [ - "value1 value2", - ] - self.compare_messages(render_output, "my_template_explicit", messages_exp) + self.assertFlowMessages( + render_output, + "my_template_default", + ["value1 default2"], + ) + self.assertFlowMessages( + render_output, + "my_template_explicit", + ["value1 value2"], + ) def test_eval(self): ci_sheet = ( @@ -515,20 +581,17 @@ def test_eval(self): "content": content, "flow": flow, } + render_output = ( + ContentIndexParser( + MockSheetReader(ci_sheet, sheet_dict), + "tests.datarowmodels.evalmodels", + ) + .parse_all() + .render() + ) - sheet_reader = MockSheetReader(ci_sheet, sheet_dict) - ci_parser = ContentIndexParser(sheet_reader, "tests.datarowmodels.evalmodels") - container = ci_parser.parse_all() - render_output = container.render() - messages_exp = [ - "hello", - "yes", - ] - self.compare_messages(render_output, "flow - id1", messages_exp) - messages_exp = [ - "hello", - ] - self.compare_messages(render_output, "flow - id2", messages_exp) + self.assertFlowMessages(render_output, "flow - id1", ["hello", "yes"]) + self.assertFlowMessages(render_output, "flow - id2", ["hello"]) def test_tags(self): ci_sheet = ( @@ -550,13 +613,15 @@ def test_tags(self): sheet_dict = { "flow": flow, } - sheet_reader = MockSheetReader(ci_sheet, sheet_dict) - ci_parser = ContentIndexParser(sheet_reader, "tests.datarowmodels.evalmodels") - container = ci_parser.parse_all() - render_output = container.render() - self.assertEqual( - self.get_flow_names(render_output), + render_output = ( + ContentIndexParser(sheet_reader, "tests.datarowmodels.evalmodels") + .parse_all() + .render() + ) + + self.assertFlowNamesEqual( + render_output, { "flow-world", "flow-t1", @@ -568,34 +633,41 @@ def test_tags(self): "flow-b1t2", }, ) - self.compare_messages(render_output, "flow-world", ["Hello World"]) - self.compare_messages(render_output, "flow-t1", ["Hello Tag1Only"]) - self.compare_messages(render_output, "flow-b1", ["Hello Bag1Only"]) - self.compare_messages(render_output, "flow-t2", ["Hello Tag2Only"]) - self.compare_messages(render_output, "flow-b2", ["Hello Bag2Only"]) - self.compare_messages(render_output, "flow-t1t2", ["Hello Tag1Tag2"]) - self.compare_messages(render_output, "flow-t1b2", ["Hello Tag1Bag2"]) - self.compare_messages(render_output, "flow-b1t2", ["Hello Bag1Tag2"]) - - tag_matcher = TagMatcher(["1", "tag1"]) - ci_parser = ContentIndexParser( - sheet_reader, "tests.datarowmodels.evalmodels", tag_matcher - ) - container = ci_parser.parse_all() - render_output = container.render() - self.assertEqual( - self.get_flow_names(render_output), - {"flow-world", "flow-t1", "flow-t2", "flow-b2", "flow-t1t2", "flow-t1b2"}, + self.assertFlowMessages(render_output, "flow-world", ["Hello World"]) + self.assertFlowMessages(render_output, "flow-t1", ["Hello Tag1Only"]) + self.assertFlowMessages(render_output, "flow-b1", ["Hello Bag1Only"]) + self.assertFlowMessages(render_output, "flow-t2", ["Hello Tag2Only"]) + self.assertFlowMessages(render_output, "flow-b2", ["Hello Bag2Only"]) + self.assertFlowMessages(render_output, "flow-t1t2", ["Hello Tag1Tag2"]) + self.assertFlowMessages(render_output, "flow-t1b2", ["Hello Tag1Bag2"]) + self.assertFlowMessages(render_output, "flow-b1t2", ["Hello Bag1Tag2"]) + + self.assertFlowNamesEqual( + ContentIndexParser( + sheet_reader, + "tests.datarowmodels.evalmodels", + TagMatcher(["1", "tag1"]), + ) + .parse_all() + .render(), + { + "flow-world", + "flow-t1", + "flow-t2", + "flow-b2", + "flow-t1t2", + "flow-t1b2", + }, ) - tag_matcher = TagMatcher(["1", "tag1", "bag1"]) - ci_parser = ContentIndexParser( - sheet_reader, "tests.datarowmodels.evalmodels", tag_matcher - ) - container = ci_parser.parse_all() - render_output = container.render() - self.assertEqual( - self.get_flow_names(render_output), + self.assertFlowNamesEqual( + ContentIndexParser( + sheet_reader, + "tests.datarowmodels.evalmodels", + TagMatcher(["1", "tag1", "bag1"]), + ) + .parse_all() + .render(), { "flow-world", "flow-t1", @@ -608,25 +680,30 @@ def test_tags(self): }, ) - tag_matcher = TagMatcher(["1", "tag1", "2", "tag2"]) - ci_parser = ContentIndexParser( - sheet_reader, "tests.datarowmodels.evalmodels", tag_matcher - ) - container = ci_parser.parse_all() - render_output = container.render() - self.assertEqual( - self.get_flow_names(render_output), - {"flow-world", "flow-t1", "flow-t2", "flow-t1t2"}, + self.assertFlowNamesEqual( + ContentIndexParser( + sheet_reader, + "tests.datarowmodels.evalmodels", + TagMatcher(["1", "tag1", "2", "tag2"]), + ) + .parse_all() + .render(), + { + "flow-world", + "flow-t1", + "flow-t2", + "flow-t1t2", + }, ) - tag_matcher = TagMatcher(["5", "tag1", "bag1"]) - ci_parser = ContentIndexParser( - sheet_reader, "tests.datarowmodels.evalmodels", tag_matcher - ) - container = ci_parser.parse_all() - render_output = container.render() - self.assertEqual( - self.get_flow_names(render_output), + self.assertFlowNamesEqual( + ContentIndexParser( + sheet_reader, + "tests.datarowmodels.evalmodels", + TagMatcher(["5", "tag1", "bag1"]), + ) + .parse_all() + .render(), { "flow-world", "flow-t1", @@ -639,149 +716,150 @@ def test_tags(self): }, ) + def assertFlowNamesEqual(self, rapidpro_export: dict, flow_names: Set[str]): + return self.assertEqual( + {flow["name"] for flow in rapidpro_export["flows"]}, + flow_names, + ) -class TestOperation(unittest.TestCase): - def test_concat(self): - # Concatenate two fresh sheets - ci_sheet = csv_join( + +class TestConcatOperation(TestCase): + def setUp(self): + simpleA = csv_join( + "ID,value1,value2", + "rowA,1A,2A", + ) + simpleB = csv_join( + "ID,value1,value2", + "rowB,1B,2B", + ) + self.sheet_dict = { + "simpleA": simpleA, + "simpleB": simpleB, + } + + def test_two_fresh_sheets(self): + self.ci_sheet = csv_join( "type,sheet_name,data_sheet,data_row_id,new_name,data_model,operation.type", "data_sheet,simpleA;simpleB,,,simpledata,SimpleRowModel,concat", ) - self.check_concat(ci_sheet) + self.check_concat() - def test_concat_implicit(self): - # Concatenate two fresh sheets - ci_sheet = csv_join( + def test_two_fresh_sheets_implictly(self): + self.ci_sheet = csv_join( "type,sheet_name,data_sheet,data_row_id,new_name,data_model,operation.type", "data_sheet,simpleA;simpleB,,,simpledata,SimpleRowModel,", ) - self.check_concat(ci_sheet) + self.check_concat() - def test_concat2(self): - # Concatenate a fresh sheet with an existing sheet - ci_sheet = csv_join( + def test_fresh_and_existing_sheets(self): + self.ci_sheet = csv_join( "type,sheet_name,data_sheet,data_row_id,new_name,data_model,operation.type", "data_sheet,simpleA,,,renamedA,SimpleRowModel,", "data_sheet,renamedA;simpleB,,,simpledata,SimpleRowModel,concat", ) - self.check_concat(ci_sheet) + self.check_concat() - def test_concat3(self): - # Concatenate two existing sheets - ci_sheet = csv_join( + def test_two_existing_sheets(self): + self.ci_sheet = csv_join( "type,sheet_name,data_sheet,data_row_id,new_name,data_model,operation.type", "data_sheet,simpleA,,,renamedA,SimpleRowModel,\n" "data_sheet,simpleB,,,renamedB,SimpleRowModel,\n" "data_sheet,renamedA;renamedB,,,simpledata,SimpleRowModel,concat\n", ) - self.check_concat(ci_sheet) + self.check_concat() - def check_concat(self, ci_sheet): - simpleA = csv_join( - "ID,value1,value2", - "rowA,1A,2A", + def check_concat(self): + parser = ContentIndexParser( + MockSheetReader(self.ci_sheet, self.sheet_dict), + "tests.datarowmodels.simplemodel", ) - simpleB = csv_join( - "ID,value1,value2", - "rowB,1B,2B", - ) - sheet_dict = { - "simpleA": simpleA, - "simpleB": simpleB, - } + datamodelA = parser.get_data_sheet_row("simpledata", "rowA") + datamodelB = parser.get_data_sheet_row("simpledata", "rowB") - sheet_reader = MockSheetReader(ci_sheet, sheet_dict) - ci_parser = ContentIndexParser(sheet_reader, "tests.datarowmodels.simplemodel") - datamodelA = ci_parser.get_data_sheet_row("simpledata", "rowA") - datamodelB = ci_parser.get_data_sheet_row("simpledata", "rowB") self.assertEqual(datamodelA.value1, "1A") self.assertEqual(datamodelA.value2, "2A") self.assertEqual(datamodelB.value1, "1B") self.assertEqual(datamodelB.value2, "2B") + +class TestOperation(TestCase): + def setUp(self): + self.simple = csv_join( + "ID,value1,value2", + "rowA,orange,fruit", + "rowB,potato,root", + "rowC,apple,fruit", + "rowD,Manioc,root", + ) + def test_filter_fresh(self): # The filter operation is referencing a sheet new (not previously parsed) sheet - ci_sheet = ( + self.ci_sheet = ( "type,sheet_name,data_sheet,data_row_id,new_name,data_model,operation\n" "data_sheet,simpleA,,,simpledata,SimpleRowModel,filter|expression;value2=='fruit'\n" # noqa: E501 ) - self.check_example1(ci_sheet) + + self.create_parser() + + self.assertRowsExistInOrder(["rowA", "rowC"]) + self.assertRowContent() def test_filter_existing(self): # The filter operation is referencing a previously parsed sheet - ci_sheet = ( + self.ci_sheet = ( "type,sheet_name,data_sheet,data_row_id,new_name,data_model,operation\n" "data_sheet,simpleA,,,,SimpleRowModel,\n" "data_sheet,simpleA,,,simpledata,SimpleRowModel,filter|expression;value2=='fruit'\n" # noqa: E501 ) - self.check_example1(ci_sheet, original="simpleA") + + self.create_parser() + + self.assertRowsExistInOrder(["rowA", "rowC"]) + self.assertRowContent() + self.assertOriginalDataNotModified("simpleA") def test_filter_existing_renamed(self): - ci_sheet = ( + self.ci_sheet = ( "type,sheet_name,data_sheet,data_row_id,new_name,data_model,operation\n" "data_sheet,simpleA,,,renamedA,SimpleRowModel,\n" "data_sheet,renamedA,,,simpledata,SimpleRowModel,filter|expression;value2=='fruit'\n" # noqa: E501 ) - self.check_example1(ci_sheet, original="renamedA") - - def check_example1(self, ci_sheet, original=None): - exp_keys = ["rowA", "rowC"] - rows = self.check_filtersort(ci_sheet, exp_keys, original) - self.assertEqual(rows["rowA"].value1, "orange") - self.assertEqual(rows["rowA"].value2, "fruit") - self.assertEqual(rows["rowC"].value1, "apple") - self.assertEqual(rows["rowC"].value2, "fruit") - - def check_filtersort(self, ci_sheet, exp_keys, original=None): - simple = csv_join( - "ID,value1,value2", - "rowA,orange,fruit", - "rowB,potato,root", - "rowC,apple,fruit", - "rowD,Manioc,root", - ) - all_keys = ["rowA", "rowB", "rowC", "rowD"] - sheet_dict = { - "simpleA": simple, - } - - sheet_reader = MockSheetReader(ci_sheet, sheet_dict) - ci_parser = ContentIndexParser(sheet_reader, "tests.datarowmodels.simplemodel") + self.create_parser() - # Ensure input data hasn't been modified - if original: - original_rows = ci_parser.get_data_sheet_rows(original) - self.assertEqual(list(original_rows.keys()), all_keys) - - # Ensure output data is as expected - rows = ci_parser.get_data_sheet_rows("simpledata") - self.assertEqual(len(rows), len(exp_keys)) - self.assertEqual(list(rows.keys()), exp_keys) - return rows + self.assertRowsExistInOrder(["rowA", "rowC"]) + self.assertRowContent() + self.assertOriginalDataNotModified("renamedA") def test_filter_fresh2(self): - ci_sheet = ( + self.ci_sheet = ( "type,sheet_name,data_sheet,data_row_id,new_name,data_model,operation\n" "data_sheet,simpleA,,,simpledata,SimpleRowModel,\"filter|expression;value1 in ['orange','apple']\"\n" # noqa: E501 ) - exp_keys = ["rowA", "rowC"] - self.check_filtersort(ci_sheet, exp_keys) + self.create_parser() + + self.assertRowsExistInOrder(["rowA", "rowC"]) def test_filter_fresh3(self): - ci_sheet = ( + self.ci_sheet = ( "type,sheet_name,data_sheet,data_row_id,new_name,data_model,operation\n" "data_sheet,simpleA,,,simpledata,SimpleRowModel,filter|expression;value1.lower() > 'd'\n" # noqa: E501 ) - exp_keys = ["rowA", "rowB", "rowD"] - self.check_filtersort(ci_sheet, exp_keys) + + self.create_parser() + + self.assertRowsExistInOrder(["rowA", "rowB", "rowD"]) def test_sort(self): - ci_sheet = ( + self.ci_sheet = ( "type,sheet_name,data_sheet,data_row_id,new_name,data_model,operation\n" "data_sheet,simpleA,,,simpledata,SimpleRowModel,sort|expression;value1.lower()\n" # noqa: E501 ) - exp_keys = ["rowC", "rowD", "rowA", "rowB"] - rows = self.check_filtersort(ci_sheet, exp_keys) + + self.create_parser() + + self.assertRowsExistInOrder(["rowC", "rowD", "rowA", "rowB"]) + rows = self.ci_parser.get_data_sheet_rows("simpledata") self.assertEqual(rows["rowA"].value1, "orange") self.assertEqual(rows["rowA"].value2, "fruit") self.assertEqual(rows["rowB"].value1, "potato") @@ -789,21 +867,52 @@ def test_sort(self): self.assertEqual(rows["rowD"].value1, "Manioc") def test_sort_existing(self): - ci_sheet = ( + self.ci_sheet = ( "type,sheet_name,data_sheet,data_row_id,new_name,data_model,operation\n" "data_sheet,simpleA,,,,SimpleRowModel,\n" "data_sheet,simpleA,,,simpledata,SimpleRowModel,sort|expression;value1.lower()\n" # noqa: E501 ) - exp_keys = ["rowC", "rowD", "rowA", "rowB"] - self.check_filtersort(ci_sheet, exp_keys, original="simpleA") + + self.create_parser() + + self.assertRowsExistInOrder(["rowC", "rowD", "rowA", "rowB"]) + self.assertOriginalDataNotModified("simpleA") def test_sort_descending(self): - ci_sheet = ( + self.ci_sheet = ( "type,sheet_name,data_sheet,data_row_id,new_name,data_model,operation\n" "data_sheet,simpleA,,,simpledata,SimpleRowModel,sort|expression;value1.lower()|order;descending\n" # noqa: E501 ) - exp_keys = ["rowB", "rowA", "rowD", "rowC"] - self.check_filtersort(ci_sheet, exp_keys) + + self.create_parser() + + self.assertRowsExistInOrder(["rowB", "rowA", "rowD", "rowC"]) + + def create_parser(self): + self.ci_parser = ContentIndexParser( + MockSheetReader(self.ci_sheet, {"simpleA": self.simple}), + "tests.datarowmodels.simplemodel", + ) + + def assertRowContent(self): + rows = self.ci_parser.get_data_sheet_rows("simpledata") + + self.assertEqual(rows["rowA"].value1, "orange") + self.assertEqual(rows["rowA"].value2, "fruit") + self.assertEqual(rows["rowC"].value1, "apple") + self.assertEqual(rows["rowC"].value2, "fruit") + + def assertRowsExistInOrder(self, exp_keys): + rows = self.ci_parser.get_data_sheet_rows("simpledata") + + self.assertEqual(len(rows), len(exp_keys)) + self.assertEqual(list(rows.keys()), exp_keys) + + def assertOriginalDataNotModified(self, name): + self.assertEqual( + list(self.ci_parser.get_data_sheet_rows(name).keys()), + ["rowA", "rowB", "rowC", "rowD"], + ) class TestModelInference(TestTemplate): @@ -820,48 +929,54 @@ def setUp(self): ",send_message,,{{custom_field.happy}} and {{custom_field.sad}}\n" ) - def check_example(self, sheet_dict): - sheet_reader = MockSheetReader(self.ci_sheet, sheet_dict) - ci_parser = ContentIndexParser(sheet_reader) - container = ci_parser.parse_all() - render_output = container.render() - self.compare_messages( - render_output, - "my_template - row1", - ["Lst 0 4", "Happy1 and Sad1"], - ) - self.compare_messages( - render_output, - "my_template - row2", - ["Lst 1 5", "Happy2 and Sad2"], - ) - def test_model_inference(self): - mydata = ( + self.mydata = ( "ID,lst.1:int,lst.2:int,custom_field.happy,custom_field.sad\n" "row1,0,4,Happy1,Sad1\n" "row2,1,5,Happy2,Sad2\n" ) - sheet_dict = { - "mydata": mydata, - "my_template": self.my_template, - } - self.check_example(sheet_dict) + + flows = self.render_flows() + + self.assertFlows(flows) def test_model_inference_alt(self): - mydata = ( + self.mydata = ( "ID,lst:List[int],custom_field.happy,custom_field.sad\n" "row1,0;4,Happy1,Sad1\n" "row2,1;5,Happy2,Sad2\n" ) + + flows = self.render_flows() + + self.assertFlows(flows) + + def render_flows(self): sheet_dict = { - "mydata": mydata, + "mydata": self.mydata, "my_template": self.my_template, } - self.check_example(sheet_dict) + return ( + ContentIndexParser(MockSheetReader(self.ci_sheet, sheet_dict)) + .parse_all() + .render() + ) -class TestParseCampaigns(unittest.TestCase): + def assertFlows(self, flows): + self.assertFlowMessages( + flows, + "my_template - row1", + ["Lst 0 4", "Happy1 and Sad1"], + ) + self.assertFlowMessages( + flows, + "my_template - row2", + ["Lst 1 5", "Happy2 and Sad2"], + ) + + +class TestParseCampaigns(TestCase): def test_parse_flow_campaign(self): ci_sheet = ( "type,sheet_name,new_name,group\n" @@ -876,13 +991,16 @@ def test_parse_flow_campaign(self): "row_id,type,from,message_text", ",send_message,start,Some text", ) - sheet_reader = MockSheetReader( - ci_sheet, {"my_campaign": my_campaign, "my_basic_flow": my_basic_flow} + ci_sheet, + { + "my_campaign": my_campaign, + "my_basic_flow": my_basic_flow, + }, ) - ci_parser = ContentIndexParser(sheet_reader) - container = ci_parser.parse_all() - render_output = container.render() + + render_output = ContentIndexParser(sheet_reader).parse_all().render() + self.assertEqual(render_output["campaigns"][0]["name"], "renamed_campaign") self.assertEqual(render_output["campaigns"][0]["group"]["name"], "My Group") event = render_output["campaigns"][0]["events"][0] @@ -892,7 +1010,8 @@ def test_parse_flow_campaign(self): self.assertEqual(event["delivery_hour"], -1) self.assertEqual(event["message"], None) self.assertEqual( - event["relative_to"], {"label": "Created On", "key": "created_on"} + event["relative_to"], + {"label": "Created On", "key": "created_on"}, ) self.assertEqual(event["start_mode"], "I") self.assertEqual(event["flow"]["name"], "my_basic_flow") @@ -909,10 +1028,12 @@ def test_parse_message_campaign(self): "150,D,M,12,Messagetext,Created On,I,\n" ) - sheet_reader = MockSheetReader(ci_sheet, {"my_campaign": my_campaign}) - ci_parser = ContentIndexParser(sheet_reader) - container = ci_parser.parse_all() - render_output = container.render() + render_output = ( + ContentIndexParser(MockSheetReader(ci_sheet, {"my_campaign": my_campaign})) + .parse_all() + .render() + ) + self.assertEqual(render_output["campaigns"][0]["name"], "my_campaign") event = render_output["campaigns"][0]["events"][0] self.assertEqual(event["event_type"], "M") @@ -938,13 +1059,15 @@ def test_duplicate_campaign(self): "my_campaign": my_campaign, "my_campaign2": my_campaign2, } - sheet_reader = MockSheetReader(ci_sheet, sheet_dict) - ci_parser = ContentIndexParser(sheet_reader) - container = ci_parser.parse_all() - render_output = container.render() + + render_output = ( + ContentIndexParser(MockSheetReader(ci_sheet, sheet_dict)) + .parse_all() + .render() + ) + self.assertEqual(render_output["campaigns"][0]["name"], "my_campaign") - event = render_output["campaigns"][0]["events"][0] - self.assertEqual(event["delivery_hour"], 6) + self.assertEqual(render_output["campaigns"][0]["events"][0]["delivery_hour"], 6) def test_ignore_campaign(self): ci_sheet = ( @@ -960,15 +1083,18 @@ def test_ignore_campaign(self): sheet_dict = { "my_campaign": my_campaign, } - sheet_reader = MockSheetReader(ci_sheet, sheet_dict) - ci_parser = ContentIndexParser(sheet_reader) - container = ci_parser.parse_all() - render_output = container.render() + + render_output = ( + ContentIndexParser(MockSheetReader(ci_sheet, sheet_dict)) + .parse_all() + .render() + ) + self.assertEqual(len(render_output["campaigns"]), 1) self.assertEqual(render_output["campaigns"][0]["name"], "my_renamed_campaign") -class TestParseTriggers(unittest.TestCase): +class TestParseTriggers(TestCase): def test_parse_triggers(self): ci_sheet = ( "type,sheet_name\n" @@ -987,15 +1113,20 @@ def test_parse_triggers(self): ",send_message,start,Some text", ) - sheet_reader = MockSheetReader( - ci_sheet, {"my_triggers": my_triggers, "my_basic_flow": my_basic_flow} + render_output = ( + ContentIndexParser( + MockSheetReader( + ci_sheet, + { + "my_triggers": my_triggers, + "my_basic_flow": my_basic_flow, + }, + ) + ) + .parse_all() + .render() ) - ci_parser = ContentIndexParser(sheet_reader) - container = ci_parser.parse_all() - render_output = container.render() - flow_uuid = render_output["flows"][0]["uuid"] - mygroup_uuid = render_output["groups"][0]["uuid"] - othergroup_uuid = render_output["groups"][1]["uuid"] + self.assertEqual(render_output["triggers"][0]["trigger_type"], "K") self.assertEqual(render_output["triggers"][1]["trigger_type"], "C") self.assertEqual(render_output["triggers"][2]["trigger_type"], "M") @@ -1012,9 +1143,14 @@ def test_parse_triggers(self): for i in range(3): self.assertIsNone(render_output["triggers"][i]["channel"]) self.assertEqual( - render_output["triggers"][i]["flow"]["name"], "my_basic_flow" + render_output["triggers"][i]["flow"]["name"], + "my_basic_flow", + ) + self.assertEqual( + render_output["triggers"][i]["flow"]["uuid"], + render_output["flows"][0]["uuid"], ) - self.assertEqual(render_output["triggers"][i]["flow"]["uuid"], flow_uuid) + mygroup_uuid = render_output["groups"][0]["uuid"] groups0 = render_output["triggers"][0]["groups"] groups1 = render_output["triggers"][1]["groups"] groups2 = render_output["triggers"][2]["exclude_groups"] @@ -1023,7 +1159,7 @@ def test_parse_triggers(self): self.assertEqual(groups1[0]["name"], "My Group") self.assertEqual(groups1[0]["uuid"], mygroup_uuid) self.assertEqual(groups1[1]["name"], "Other Group") - self.assertEqual(groups1[1]["uuid"], othergroup_uuid) + self.assertEqual(groups1[1]["uuid"], render_output["groups"][1]["uuid"]) self.assertEqual(groups2[0]["name"], "My Group") self.assertEqual(groups2[0]["uuid"], mygroup_uuid) @@ -1034,11 +1170,10 @@ def test_parse_triggers_without_flow(self): "K,the word,my_basic_flow,My Group,,\n" ) - sheet_reader = MockSheetReader(ci_sheet, {"my_triggers": my_triggers}) - ci_parser = ContentIndexParser(sheet_reader) - container = ci_parser.parse_all() with self.assertRaises(RapidProTriggerError): - container.render() + ContentIndexParser( + MockSheetReader(ci_sheet, {"my_triggers": my_triggers}) + ).parse_all().render() def test_ignore_triggers(self): ci_sheet = ( @@ -1054,12 +1189,18 @@ def test_ignore_triggers(self): "row_id,type,from,message_text", ",send_message,start,Some text", ) - sheet_reader = MockSheetReader( - ci_sheet, {"my_triggers": my_triggers, "my_basic_flow": my_basic_flow} + + render_output = ( + ContentIndexParser( + MockSheetReader( + ci_sheet, + {"my_triggers": my_triggers, "my_basic_flow": my_basic_flow}, + ) + ) + .parse_all() + .render() ) - ci_parser = ContentIndexParser(sheet_reader) - container = ci_parser.parse_all() - render_output = container.render() + self.assertEqual(len(render_output["triggers"]), 0) @@ -1067,77 +1208,106 @@ class TestParseFromFile(TestTemplate): def setUp(self): self.input_dir = TESTS_ROOT / "input/example1" - def compare_to_expected(self, render_output): - self.compare_messages(render_output, "my_basic_flow", ["Some text"]) - self.compare_messages( - render_output, "my_template - row1", ["Value1", "Happy1 and Sad1"] - ) - self.compare_messages( - render_output, "my_template - row2", ["Value2", "Happy2 and Sad2"] - ) - self.assertEqual(render_output["campaigns"][0]["name"], "my_campaign") - self.assertEqual(render_output["campaigns"][0]["group"]["name"], "My Group") - self.assertEqual( - render_output["campaigns"][0]["events"][0]["flow"]["name"], "my_basic_flow" - ) - self.assertEqual( - render_output["campaigns"][0]["events"][0]["flow"]["uuid"], - render_output["flows"][2]["uuid"], + def test_example1_csv(self): + flows = ( + ContentIndexParser( + CSVSheetReader(self.input_dir / "csv_workbook"), + "tests.input.example1.nestedmodel", + ) + .parse_all() + .render() ) - def check_example1(self, ci_parser): - container = ci_parser.parse_all() - render_output = container.render() - self.compare_to_expected(render_output) - - def test_example1_csv(self): - # Same test as test_generate_flows but with csvs - sheet_reader = CSVSheetReader(self.input_dir / "csv_workbook") - ci_parser = ContentIndexParser(sheet_reader, "tests.input.example1.nestedmodel") - self.check_example1(ci_parser) + self.assertFlows(flows) def test_example1_csv_composite(self): - # Same test as test_generate_flows but with csvs - sheet_reader = CSVSheetReader(self.input_dir / "csv_workbook") - reader = CompositeSheetReader([sheet_reader]) - ci_parser = ContentIndexParser(reader, "tests.input.example1.nestedmodel") - self.check_example1(ci_parser) + flows = ( + ContentIndexParser( + CompositeSheetReader([CSVSheetReader(self.input_dir / "csv_workbook")]), + "tests.input.example1.nestedmodel", + ) + .parse_all() + .render() + ) + + self.assertFlows(flows) def test_example1_xlsx(self): - # Same test as above - sheet_reader = XLSXSheetReader(self.input_dir / "content_index.xlsx") - ci_parser = ContentIndexParser(sheet_reader, "tests.input.example1.nestedmodel") - self.check_example1(ci_parser) + flows = ( + ContentIndexParser( + XLSXSheetReader(self.input_dir / "content_index.xlsx"), + "tests.input.example1.nestedmodel", + ) + .parse_all() + .render() + ) + + self.assertFlows(flows) def test_example1_xlsx_composite(self): - # Same test as above - sheet_reader = XLSXSheetReader(self.input_dir / "content_index.xlsx") - reader = CompositeSheetReader([sheet_reader]) - ci_parser = ContentIndexParser(reader, "tests.input.example1.nestedmodel") - self.check_example1(ci_parser) + flows = ( + ContentIndexParser( + CompositeSheetReader( + [XLSXSheetReader(self.input_dir / "content_index.xlsx")] + ), + "tests.input.example1.nestedmodel", + ) + .parse_all() + .render() + ) + self.assertFlows(flows) -class TestMultiFile(TestTemplate): - def check(self, ci_parser, flow_name, messages_exp): - container = ci_parser.parse_all() - render_output = container.render() - self.compare_messages(render_output, flow_name, messages_exp) + def assertFlows(self, flows): + self.assertFlowMessages( + flows, + "my_basic_flow", + ["Some text"], + ) + self.assertFlowMessages( + flows, + "my_template - row1", + ["Value1", "Happy1 and Sad1"], + ) + self.assertFlowMessages( + flows, + "my_template - row2", + ["Value2", "Happy2 and Sad2"], + ) + self.assertEqual( + flows["campaigns"][0]["name"], + "my_campaign", + ) + self.assertEqual( + flows["campaigns"][0]["group"]["name"], + "My Group", + ) + self.assertEqual( + flows["campaigns"][0]["events"][0]["flow"]["name"], + "my_basic_flow", + ) + self.assertEqual( + flows["campaigns"][0]["events"][0]["flow"]["uuid"], + flows["flows"][2]["uuid"], + ) + +class TestMultiFile(TestTemplate): def test_minimal(self): - self.run_minimal() + ci_sheet = csv_join( + "type,sheet_name", + "template_definition,template", + ) + self.run_minimal(ci_sheet) def test_minimal_singleindex(self): - self.run_minimal(True) + self.run_minimal(ci_sheet=None) - def run_minimal(self, singleindex=False): + def run_minimal(self, ci_sheet): ci_sheet1 = csv_join( "type,sheet_name", "create_flow,template", ) - ci_sheet2 = csv_join( - "type,sheet_name", - "template_definition,template", - ) template = csv_join( "row_id,type,from,message_text", ",send_message,start,Hello!", @@ -1146,21 +1316,19 @@ def run_minimal(self, singleindex=False): "template": template, } sheet_reader1 = MockSheetReader(ci_sheet1, name="mock_1") + sheet_reader2 = MockSheetReader(ci_sheet, sheet_dict2, name="mock_2") - sheet_reader2 = MockSheetReader( - None if singleindex else ci_sheet2, - sheet_dict2, - name="mock_2", - ) - - self.check( - ContentIndexParser(CompositeSheetReader([sheet_reader1, sheet_reader2])), + self.assertFlowMessages( + ContentIndexParser(CompositeSheetReader([sheet_reader1, sheet_reader2])) + .parse_all() + .render(), "template", ["Hello!"], ) - - self.check( - ContentIndexParser(CompositeSheetReader([sheet_reader2, sheet_reader1])), + self.assertFlowMessages( + ContentIndexParser(CompositeSheetReader([sheet_reader2, sheet_reader1])) + .parse_all() + .render(), "template", ["Hello!"], ) @@ -1201,22 +1369,33 @@ def test_with_model(self): } sheet_reader1 = MockSheetReader(ci_sheet1, sheet_dict1, name="mock_1") sheet_reader2 = MockSheetReader(ci_sheet2, sheet_dict2, name="mock_2") - ci_parser = ContentIndexParser( - sheet_reader=CompositeSheetReader([sheet_reader1, sheet_reader2]), - user_data_model_module_name="tests.datarowmodels.minimalmodel", + + flows = ( + ContentIndexParser( + sheet_reader=CompositeSheetReader([sheet_reader1, sheet_reader2]), + user_data_model_module_name="tests.datarowmodels.minimalmodel", + ) + .parse_all() + .render() ) - self.check(ci_parser, "template - a", ["hi georg"]) - self.check(ci_parser, "template - b", ["hi chiara"]) - ci_parser = ContentIndexParser( - sheet_reader=CompositeSheetReader([sheet_reader2, sheet_reader1]), - user_data_model_module_name="tests.datarowmodels.minimalmodel", + self.assertFlowMessages(flows, "template - a", ["hi georg"]) + self.assertFlowMessages(flows, "template - b", ["hi chiara"]) + + flows = ( + ContentIndexParser( + sheet_reader=CompositeSheetReader([sheet_reader2, sheet_reader1]), + user_data_model_module_name="tests.datarowmodels.minimalmodel", + ) + .parse_all() + .render() ) - self.check(ci_parser, "template - a", ["hello georg"]) - self.check(ci_parser, "template - b", ["hello chiara"]) + + self.assertFlowMessages(flows, "template - a", ["hello georg"]) + self.assertFlowMessages(flows, "template - b", ["hello chiara"]) -class TestSaveAsDict(unittest.TestCase): +class TestSaveAsDict(TestCase): def test_save_as_dict(self): self.maxDiff = None ci_sheet = ( @@ -1238,16 +1417,17 @@ def test_save_as_dict(self): "row_id,type,from,message_text", ",send_message,start,Some text", ) - sheet_dict = { "simpledata": simpledata, "my_basic_flow": my_basic_flow, "nesteddata": nesteddata, } - sheet_reader = MockSheetReader(ci_sheet, sheet_dict) - ci_parser = ContentIndexParser(sheet_reader, "tests.datarowmodels.nestedmodel") - output = ci_parser.data_sheets_to_dict() + output = ContentIndexParser( + MockSheetReader(ci_sheet, sheet_dict), + "tests.datarowmodels.nestedmodel", + ).data_sheets_to_dict() + output["meta"].pop("version") exp = { "meta": {