refac: Use exceptAll instead of subtract in assert when comparing dat…

…aframes (#3050)

* avoid many counts

* avoid many counts

* ...

* re-introduce count check

* sample.yml

* use exceptAll instead of subtract in assert

source/databricks/calculation_engine/tests/helpers/data_frame_utils.py

@@ -31,8 +31,8 @@ def set_column(
 
 
 def assert_dataframes_equal(actual: DataFrame, expected: DataFrame) -> None:
-    actual_excess = actual.subtract(expected)
-    expected_excess = expected.subtract(actual)
+    actual_excess = actual.exceptAll(expected)
+    expected_excess = expected.exceptAll(actual)
 
     # do the count once here to avoid materializing multiple times
     actual_excess_count = actual_excess.count()
@@ -155,20 +155,6 @@ def assert_dataframe_and_schema(
         expected.subtract(actual).show(3000, False)
         raise
 
-    try:
-        assert actual.count() == expected.count()
-    except AssertionError:
-
-        if (
-            not feature_tests_configuration.show_columns_when_actual_and_expected_are_equal
-        ):
-            actual, expected = drop_columns_if_the_same(actual, expected)
-
-        print(
-            f"NUMBER OF ROWS MISMATCH: Actual: {actual.count()}, Expected: {expected.count()}"
-        )
-        raise
-
 
 def drop_columns_if_the_same(df1: DataFrame, df2: DataFrame) -> (DataFrame, DataFrame):
     column_names = df1.columns