Skip to content

Commit

Permalink
trying to fix testing
Browse files Browse the repository at this point in the history
  • Loading branch information
bvonodiripsa committed Jul 31, 2024
1 parent b39218b commit f058cb2
Show file tree
Hide file tree
Showing 2 changed files with 11 additions and 2 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -39,20 +39,29 @@ def __init__(self, *args, **kwargs):
.cache()
)


def test_e5_Embedding(self):
self._assert_input(self.sentenceDataFrame)
transformed = self.e5Transformer.transform(self.sentenceDataFrame).cache()
self._assert_input(self.transformed)
self._assert_embedding_df_size(self.sentenceDataFrame, transformed)
self._assert_embedding_embedding_size(transformed, self.e5Size)

def test_miniLM_Embedding(self):
self._assert_input(self.sentenceDataFrame)
transformed = self.miniLMTransformer.transform(self.sentenceDataFrame).cache()
self._assert_input(self.transformed)
self._assert_embedding_df_size(self.sentenceDataFrame, transformed)
self._assert_embedding_embedding_size(transformed, self.miniLMSize)

def _assert_input(self, input):
# Use assert to check if the result is a DataFrame
testDf = self.sentenceDataFrame
assert isinstance(testDf, pyspark.sql.DataFrame), "The input is not a DataFrame."

def _assert_embedding_embedding_size(self, transformed, expected_size):
# Debugging to check the type
collected_data = transformed.collect()

for row in collected_data:
embeddings_array = row["embeddings"]
size = len(embeddings_array)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -468,7 +468,7 @@
"\n",
"This is the comparison dureation results on 10 T4 GPU nodes for both approaches:\n",
"\n",
"<img src=\"https://mmlspark.blob.core.windows.net/graphics/Documentation/knn_comparison.png\" alt=\"KNN Comparison\" width=\"50%\">\n",
"![KNN Comparison](https://mmlspark.blob.core.windows.net/graphics/Documentation/knn_comparison.png)\n",
"\n",
"\n"
]
Expand Down

0 comments on commit f058cb2

Please sign in to comment.