Skip to content

Commit

Permalink
Fix MD duplicated Node id from multiple docs (#10564)
Browse files Browse the repository at this point in the history
* Fix MD duplicated Node id from multiple docs

* cr
  • Loading branch information
hatianzhang authored Feb 11, 2024
1 parent 60b75cb commit e3a169b
Showing 1 changed file with 16 additions and 6 deletions.
22 changes: 16 additions & 6 deletions llama_index/node_parser/relational/markdown_element.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,9 @@ def class_name(cls) -> str:
def get_nodes_from_node(self, node: TextNode) -> List[BaseNode]:
"""Get nodes from node."""
elements = self.extract_elements(
node.get_content(), table_filters=[self.filter_table]
node.get_content(),
table_filters=[self.filter_table],
node_id=node.id_,
)
table_elements = self.get_table_elements(elements)
# extract summaries over table elements
Expand All @@ -59,8 +61,13 @@ def get_nodes_from_node(self, node: TextNode) -> List[BaseNode]:
return self.get_nodes_from_elements(elements)

def extract_elements(
self, text: str, table_filters: Optional[List[Callable]] = None, **kwargs: Any
self,
text: str,
node_id: Optional[str] = None,
table_filters: Optional[List[Callable]] = None,
**kwargs: Any,
) -> List[Element]:
# get node id for each node so that we can avoid using the same id for different nodes
"""Extract elements from text."""
lines = text.split("\n")
currentElement = None
Expand Down Expand Up @@ -168,27 +175,30 @@ def extract_elements(
table = md_to_df(element.element)

elements[idx] = Element(
id=f"id_{idx}", type="table", element=element, table=table
id=f"id_{node_id}_{idx}" if node_id else f"id_{idx}",
type="table",
element=element,
table=table,
)
else:
# for non-perfect tables, we will store the raw text
# and give it a different type to differentiate it from perfect tables
elements[idx] = Element(
id=f"id_{idx}",
id=f"id_{node_id}_{idx}" if node_id else f"id_{idx}",
type="table_text",
element=element.element,
# table=table
)
else:
elements[idx] = Element(
id=f"id_{idx}",
id=f"id_{node_id}_{idx}" if node_id else f"id_{idx}",
type="text",
element=element.element,
)
else:
# if the element is not a table, keep it as to text
elements[idx] = Element(
id=f"id_{idx}",
id=f"id_{node_id}_{idx}" if node_id else f"id_{idx}",
type="text",
element=element.element,
)
Expand Down

0 comments on commit e3a169b

Please sign in to comment.