analyse_model.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Example model loading and analysis\n",
    "This notebook demonstrates how to load a pretrained model and perform further analysis and checks."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Collect imports\n",
    "from typing import Dict, Tuple, Any\n",
    "from pprint import pprint\n",
    "import json\n",
    "\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "import mlflow\n",
    "from mlflow.tracking import MlflowClient\n",
    "\n",
    "from components.dnf_layer import WeightedDNF\n",
    "\n",
    "import utils.analysis\n",
    "import utils.clingo\n",
    "\n",
    "np.set_printoptions(suppress=True, precision=5, linewidth=120)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['relsgame-predinet-20210429-121731', 'relsgame-dnf-20210429-121731', 'gendnf-ilp-20210503-164950', 'gendnf-deep-20210428-230630', 'relsgame-recon_dnf-20210511-215943']\n"
     ]
    }
   ],
   "source": [
    "# Setup mlflow client\n",
    "mlclient = MlflowClient(tracking_uri=\"http://localhost:8888\") # Emptry string for local\n",
    "exp_list = [x.name for x in mlclient.list_experiments()]\n",
    "print(exp_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Experiment Name: relsgame-recon_dnf-20210511-215943\n",
      "Collecting artifacts for run: 8f7f23d93d94487fbdc5ea36c203c335\n",
      "{'data': {'metrics': {'epoch': 149.0,\n",
      "                      'test_hexos_loss': 0.019202135503292084,\n",
      "                      'test_hexos_nullary_acc': 0.9950000047683716,\n",
      "                      'test_hexos_nullary_loss': 0.012011142447590828,\n",
      "                      'test_hexos_recon_loss': 0.007190990261733532,\n",
      "                      'test_hexos_recon_mae': 0.029814941808581352,\n",
      "                      'test_pentos_loss': 0.027659451588988304,\n",
      "                      'test_pentos_nullary_acc': 0.9919999837875366,\n",
      "                      'test_pentos_nullary_loss': 0.026043063029646873,\n",
      "                      'test_pentos_recon_loss': 0.0016163942636922002,\n",
      "                      'test_pentos_recon_mae': 0.011510365642607212,\n",
      "                      'test_stripes_loss': 0.15420617163181305,\n",
      "                      'test_stripes_nullary_acc': 0.9909999966621399,\n",
      "                      'test_stripes_nullary_loss': 0.03997662290930748,\n",
      "                      'test_stripes_recon_loss': 0.11422955989837646,\n",
      "                      'test_stripes_recon_mae': 0.13385647535324097,\n",
      "                      'time': 14.224395036697388,\n",
      "                      'train_dnf_layer/success_threshold': 1.0,\n",
      "                      'train_loss': 0.027398675680160522,\n",
      "                      'train_nullary_acc': 0.9911114573478699,\n",
      "                      'train_nullary_loss': 0.02472049370408058,\n",
      "                      'train_object_selector/temperature': 0.009999999776482582,\n",
      "                      'train_recon_loss': 0.002678174991160631,\n",
      "                      'train_recon_mae': 0.013074344024062157,\n",
      "                      'validation_loss': 0.030458806082606316,\n",
      "                      'validation_nullary_acc': 0.9919999837875366,\n",
      "                      'validation_nullary_loss': 0.02896002121269703,\n",
      "                      'validation_recon_loss': 0.0014987797476351261,\n",
      "                      'validation_recon_mae': 0.011478456668555737},\n",
      "          'params': {'config_json': 'data/experiments.json.6218a9cd0485c46c1af809a6e88e76b9',\n",
      "                     'data_dir': 'data',\n",
      "                     'dataset_name': 'relsgame',\n",
      "                     'debug': 'False',\n",
      "                     'dnf_image_classifier_hidden_arities': '[]',\n",
      "                     'dnf_image_classifier_hidden_layer_name': 'WeightedDNF',\n",
      "                     'dnf_image_classifier_hidden_num_conjuncts': '4',\n",
      "                     'dnf_image_classifier_hidden_num_total_variables': '2',\n",
      "                     'dnf_image_classifier_hidden_recursive': 'False',\n",
      "                     'dnf_image_classifier_image_activation': 'relu',\n",
      "                     'dnf_image_classifier_image_hidden_size': '32',\n",
      "                     'dnf_image_classifier_image_layer_name': 'RelationsGameImageInput',\n",
      "                     'dnf_image_classifier_image_with_position': 'True',\n",
      "                     'dnf_image_classifier_inference_arities': '[]',\n",
      "                     'dnf_image_classifier_inference_layer_name': 'WeightedDNF',\n",
      "                     'dnf_image_classifier_inference_num_conjuncts': '8',\n",
      "                     'dnf_image_classifier_inference_num_total_variables': '3',\n",
      "                     'dnf_image_classifier_inference_recursive': 'False',\n",
      "                     'dnf_image_classifier_iterations': '1',\n",
      "                     'dnf_image_classifier_object_feat_activation': 'tanh',\n",
      "                     'dnf_image_classifier_object_feat_binary_size': '16',\n",
      "                     'dnf_image_classifier_object_feat_layer_name': 'LinearObjectFeatures',\n",
      "                     'dnf_image_classifier_object_feat_unary_size': '8',\n",
      "                     'dnf_image_classifier_object_sel_initial_temperature': '0.5',\n",
      "                     'dnf_image_classifier_object_sel_layer_name': 'RelaxedObjectSelection',\n",
      "                     'dnf_image_classifier_object_sel_num_select': '3',\n",
      "                     'dnf_rule_learner_inference_arities': '[0]',\n",
      "                     'dnf_rule_learner_inference_layer_name': 'WeightedDNF',\n",
      "                     'dnf_rule_learner_inference_num_conjuncts': '8',\n",
      "                     'dnf_rule_learner_inference_num_total_variables': '2',\n",
      "                     'dnf_rule_learner_inference_recursive': 'False',\n",
      "                     'eval_every': '200',\n",
      "                     'experiment_name': 'relsgame-recon_dnf-20210511-215943',\n",
      "                     'gendnf_batch_size': '64',\n",
      "                     'gendnf_gen_size': '10000',\n",
      "                     'gendnf_input_noise_probability': '0.0',\n",
      "                     'gendnf_num_binary': '8',\n",
      "                     'gendnf_num_conjuncts': '4',\n",
      "                     'gendnf_num_nullary': '6',\n",
      "                     'gendnf_num_objects': '4',\n",
      "                     'gendnf_num_unary': '7',\n",
      "                     'gendnf_num_variables': '3',\n",
      "                     'gendnf_return_numpy': 'False',\n",
      "                     'gendnf_rng_seed': '42',\n",
      "                     'gendnf_target_arity': '0',\n",
      "                     'gendnf_test_size': '1000',\n",
      "                     'gendnf_train_size': '1000',\n",
      "                     'gendnf_validation_size': '1000',\n",
      "                     'learning_rate': '0.001',\n",
      "                     'max_steps': '30000',\n",
      "                     'mlp_image_classifier_hidden_activations': \"['relu']\",\n",
      "                     'mlp_image_classifier_hidden_sizes': '[120, 120]',\n",
      "                     'mlp_image_classifier_image_activation': 'relu',\n",
      "                     'mlp_image_classifier_image_hidden_size': '32',\n",
      "                     'mlp_image_classifier_image_layer_name': 'RelationsGameImageInput',\n",
      "                     'mlp_image_classifier_image_with_position': 'False',\n",
      "                     'model_name': 'dnf_image_classifier',\n",
      "                     'nickname': 'recon_dnf_image_classifier',\n",
      "                     'predinet_heads': '7',\n",
      "                     'predinet_image_activation': 'relu',\n",
      "                     'predinet_image_hidden_size': '32',\n",
      "                     'predinet_image_layer_name': 'RelationsGameImageInput',\n",
      "                     'predinet_image_with_position': 'False',\n",
      "                     'predinet_key_size': '4',\n",
      "                     'predinet_output_hidden_size': '8',\n",
      "                     'predinet_relations': '16',\n",
      "                     'relsgame_batch_size': '64',\n",
      "                     'relsgame_noise_stddev': '0.01',\n",
      "                     'relsgame_output_type': 'label_and_image',\n",
      "                     'relsgame_rng_seed': '42',\n",
      "                     'relsgame_task_nickname': 'between',\n",
      "                     'relsgame_tasks': \"['between']\",\n",
      "                     'relsgame_test_size': '1000',\n",
      "                     'relsgame_train_size': '1000',\n",
      "                     'relsgame_validation_size': '1000',\n",
      "                     'relsgame_with_augmentation': 'True',\n",
      "                     'run_count': '4',\n",
      "                     'slotae_image_activation': 'relu',\n",
      "                     'slotae_image_hidden_size': '32',\n",
      "                     'slotae_image_layer_name': 'RelationsGameImageInput',\n",
      "                     'slotae_image_with_position': 'False',\n",
      "                     'tracking_uri': 'http://muli.doc.ic.ac.uk:8888',\n",
      "                     'train_type': 'deep'},\n",
      "          'tags': {'Err': 'data/outs/78-161.err',\n",
      "                   'GlobalJobId': 'muli.doc.ic.ac.uk#78.161#1620766865',\n",
      "                   'Out': 'data/outs/78-161.out',\n",
      "                   'UserLog': '/homes/nuric/phd/pix2rule/data/outs/78-161.log',\n",
      "                   'config_hash': '87ed0716fc21afcff23eeaf7d98108d2',\n",
      "                   'hostname': 'edge26.doc.ic.ac.uk',\n",
      "                   'mlflow.source.git.commit': 'a658c9444c8cfce432718229c2f56cf2bac5a377',\n",
      "                   'mlflow.source.name': 'train.py',\n",
      "                   'mlflow.source.type': 'LOCAL',\n",
      "                   'mlflow.user': 'nuric',\n",
      "                   'num_params': '62283'}},\n",
      " 'info': {'artifact_uri': '/homes/nuric/phd/pix2rule/data/artifacts/47/8f7f23d93d94487fbdc5ea36c203c335/artifacts',\n",
      "          'end_time': 1620769066624,\n",
      "          'experiment_id': '47',\n",
      "          'lifecycle_stage': 'active',\n",
      "          'run_id': '8f7f23d93d94487fbdc5ea36c203c335',\n",
      "          'run_uuid': '8f7f23d93d94487fbdc5ea36c203c335',\n",
      "          'start_time': 1620766972588,\n",
      "          'status': 'FINISHED',\n",
      "          'user_id': 'nuric'}}\n",
      "[<FileInfo: file_size=5142, is_dir=False, path='model_summary.txt'>,\n",
      " <FileInfo: file_size=None, is_dir=True, path='models'>,\n",
      " <FileInfo: file_size=122790, is_dir=False, path='pruning_info.json'>,\n",
      " <FileInfo: file_size=834, is_dir=False, path='task_description.json'>,\n",
      " <FileInfo: file_size=591388, is_dir=False, path='test_hexos_report.npz'>,\n",
      " <FileInfo: file_size=587954, is_dir=False, path='test_pentos_report.npz'>,\n",
      " <FileInfo: file_size=600320, is_dir=False, path='test_stripes_report.npz'>,\n",
      " <FileInfo: file_size=645528, is_dir=False, path='train_report.npz'>,\n",
      " <FileInfo: file_size=588160, is_dir=False, path='validation_report.npz'>]\n"
     ]
    }
   ],
   "source": [
    "# Gather a specific run\n",
    "#exp_name = exp_list[-1]\n",
    "exp_name = \"relsgame-recon_dnf-20210511-215943\"\n",
    "print(\"Experiment Name:\", exp_name)\n",
    "#all_runs = [r.info.run_id for r in mlclient.search_runs(mlclient.get_experiment_by_name(exp_name).experiment_id)]\n",
    "#print(all_runs)\n",
    "#run_id = all_runs[-1]\n",
    "run_id = \"8f7f23d93d94487fbdc5ea36c203c335\"\n",
    "print(\"Collecting artifacts for run:\", run_id)\n",
    "mlrun = mlclient.get_run(run_id)\n",
    "pprint(mlrun.to_dictionary())\n",
    "pprint(mlclient.list_artifacts(run_id))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['test_hexos', 'test_pentos', 'test_stripes', 'train', 'validation'])"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load the dataset batches for quick evaluation\n",
    "npz_files = [artifact.path for artifact in mlclient.list_artifacts(run_id) if artifact.path.endswith('.npz')]\n",
    "data_reports = {'_'.join(fname.split('_')[:-1]): utils.analysis.load_artifact(run_id, fname, mlclient) for fname in npz_files}\n",
    "# The data format is a tuple of dictionaries\n",
    "datasets = dict()\n",
    "for dname, dreport in data_reports.items():\n",
    "    # dname is train, test_hexos etc.\n",
    "    # dreport contians reported batches and model outputs\n",
    "    # such as attention maps etc.\n",
    "    input_dict, output_dict = dict(), dict()\n",
    "    for k, v in dreport.items():\n",
    "        # in_image, out_label etc.\n",
    "        if k.startswith('in'):\n",
    "            input_dict[k[3:]] = v\n",
    "        elif k.startswith('out'):\n",
    "            output_dict[k[4:]] = v\n",
    "    datasets[dname] = tf.data.Dataset.from_tensor_slices((input_dict, output_dict)).batch(v.shape[0])\n",
    "# Note that for the relations game datasets, the train is a repeated datasets that loops forever\n",
    "datasets.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Compiled the loaded model, but the compiled metrics have yet to be built. `model.compile_metrics` will be empty until you train or evaluate the model.\n",
      "Number of parameters: 62283\n"
     ]
    }
   ],
   "source": [
    "# Let's load the checkpointed model\n",
    "models_dir = mlclient.download_artifacts(run_id, 'models')\n",
    "dnf_model = tf.keras.models.load_model(models_dir + '/latest_model')\n",
    "print(\"Number of parameters:\", dnf_model.count_params())\n",
    "dnf_model.compile(loss={'label': tf.keras.losses.BinaryCrossentropy(from_logits=True)}, metrics={'label': tf.keras.metrics.BinaryAccuracy('acc', threshold=0.0)})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1/1 [==============================] - 0s 417ms/step - loss: 0.0026 - nullary_loss: 0.0026 - nullary_acc: 1.0000\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'loss': 0.002586615039035678,\n",
       " 'nullary_loss': 0.002586615039035678,\n",
       " 'nullary_acc': 1.0}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Lets evaluate the model on the test hexos batch\n",
    "dnf_model.evaluate(datasets['test_hexos'], verbose=1, return_dict=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['image', 'label'])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# We can also ask it for the predictions, note if it is an image reconstruction model\n",
    "# we will obtain {'image': (batch_size, ...), 'label': (batch_size,)}, otherwise just label\n",
    "dnf_model.predict(datasets['test_hexos'], verbose=0).keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['in_image',\n",
       " 'out_image',\n",
       " 'out_label',\n",
       " 'object_sel_object_scores',\n",
       " 'object_sel_object_atts',\n",
       " 'object_sel_objects',\n",
       " 'report_layer_unary',\n",
       " 'report_layer_binary',\n",
       " 'report_layer_nullary',\n",
       " 'facts0_nullary',\n",
       " 'facts0_unary',\n",
       " 'facts0_binary',\n",
       " 'facts0_or_kernel',\n",
       " 'facts0_conjuncts',\n",
       " 'facts0_and_kernel',\n",
       " 'recon_combined',\n",
       " 'recon_recon_masks',\n",
       " 'recon_reconstructions',\n",
       " 'prediction_image',\n",
       " 'prediction_label']"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# These are the all the intermediate reports, such as attention maps, features etc.\n",
    "data_reports['test_hexos'].files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:ylabel='Count'>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAD4CAYAAAAD6PrjAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Z1A+gAAAACXBIWXMAAAsTAAALEwEAmpwYAAASB0lEQVR4nO3de4xc5X3G8efZXe/aXtMC8ta1uNS0taJS2pBoQ7kpItAUg9pCIsIt4A2XmJRLQYloofyR9o9IkUIIEGITUwimIVCSQIFwNWBAKNwMooRrY6VY2AJmkzRg79hre/bXP/bM6/F6dndsz5mzl+9HGs3Me86c+Z2x9zzznvfMOY4IAQAgSW1FFwAAmDgIBQBAQigAABJCAQCQEAoAgKSj6AL2xNy5c2PBggVFlwEAk8pLL73064joqTdtUofCggULtHr16qLLAIBJxfba0aax+wgAkBAKAICEUAAAJIQCACAhFAAACaEAAEgIBQBAQigAABJCARhDRGhgYEBcdwTTBaEAjKFcLuu06x5WuVwuuhSgJQgFYBwdnTOLLgFoGUIBAJAQCgCAhFAAACSEAgAgIRQAAAmhAABICAUAQEIoAAASQgEAkBAKAICEUAAAJIQCACAhFAAACaEAAEgIBQBAQigAABJCAQCQEAoAgCS3ULB9gO1Vtt+w/brtS7P2fW2vtP3L7H6frN22r7e9xvartj+ZV20AgPry7Clsk/S1iDhY0uGSLrJ9sKQrJD0eEQslPZ49l6QTJC3MbkskLcuxNgBAHbmFQkS8FxEvZ483SHpT0n6STpK0IptthaSTs8cnSbothj0naW/b8/OqDwCws5aMKdheIOkTkp6XNC8i3ssmvS9pXvZ4P0nv1rxsXdY2cllLbK+2vbq/vz+/ogFgGso9FGzPkfRTSZdFxEe10yIiJMWuLC8ilkdEb0T09vT0NLFSAECuoWB7hoYD4faIuDtr/qC6Wyi7L2Xt6yUdUPPy/bM2AECL5Hn0kSXdLOnNiLimZtJ9kvqyx32S7q1pX5wdhXS4pA9rdjMBAFqgI8dlHyXpbEm/sP1K1vYvkr4p6S7b50laK+nUbNqDkk6UtEZSWdI5OdYGAKgjt1CIiGckeZTJx9WZPyRdlFc9AIDx8YtmAEBCKAAAEkIBAJAQCgCAhFAAACSEAgAgIRQAAAmhAABICAUAQEIoAAASQgEAkBAKAICEUAAAJIQCACAhFAAACaEAAEgIBQBAQigAABJCAQCQEAoAgIRQAAAkhAIAICEUAAAJoQAASAgFAEBCKAAAEkIBAJAQCgCAhFAAACSEAgAgIRQAAAmhAABICAUAQEIoAAASQgEAkBAKAICEUAAAJLmFgu1bbJdsv1bT9q+219t+JbudWDPtSttrbL9t+/i86gIAjC7PnsKtkhbVaf9ORBya3R6UJNsHSzpd0p9nr1lquz3H2gAAdeQWChHxtKTfNjj7SZLujIjBiPhfSWskHZZXbQCA+ooYU7jY9qvZ7qV9srb9JL1bM8+6rG0ntpfYXm17dX9/f961AsC00upQWCbpTyQdKuk9Sd/e1QVExPKI6I2I3p6eniaXBwDTW0tDISI+iIhKRAxJuknbdxGtl3RAzaz7Z20AgBZqaSjYnl/z9HOSqkcm3SfpdNtdtg+StFDSC62sDQAgdeS1YNt3SDpG0lzb6yR9XdIxtg+VFJLekXSBJEXE67bvkvSGpG2SLoqISl61AQDqyy0UIuKMOs03jzH/NyR9I696AADj4xfNAICEUAAAJIQCACAhFAAACaEAAEgIBQBAQigAABJCAQCQEAoAgIRQAAAkhAIAIGkoFGwf1UgbAGBya7Sn8N0G2wAAk9iYZ0m1fYSkIyX12P5qzaTfk9SeZ2EAgNYb79TZnZLmZPPtVdP+kaRT8ioKAFCMMUMhIp6S9JTtWyNibYtqAiaEiNDAwEDRZQAt1ehFdrpsL5e0oPY1EXFsHkUBE0G5XNbipSs1Y/bvF10K0DKNhsKPJd0o6d8lcZlMTBsdnTOLLgFoqUZDYVtELMu1EgBA4Ro9JPV+2xfanm973+ot18oAAC3XaE+hL7u/vKYtJP1xc8sBABSpoVCIiIPyLgQAULyGQsH24nrtEXFbc8sBABSp0d1Hn6p5PFPScZJelkQoAMAU0ujuo0tqn9veW9KdeRQEACjO7p46e0AS4wwAMMU0OqZwv4aPNpKGT4T3Z5LuyqsoAEAxGh1TuLrm8TZJayNiXQ71AAAK1NDuo+zEeG9p+Eyp+0jakmdRAIBiNHrltVMlvSDpC5JOlfS8bU6dDQBTTKO7j66S9KmIKEmS7R5Jj0n6SV6FAQBar9Gjj9qqgZD5zS68FgAwSTTaU3jY9iOS7sienybpwXxKAgAUZbxrNP+ppHkRcbntz0s6Opv0rKTb8y4OANBa4/UUrpV0pSRFxN2S7pYk23+RTfu7HGsDALTYeOMC8yLiFyMbs7YFuVQEACjMeKGw9xjTZjWxDgDABDBeKKy2/eWRjbbPl/TSWC+0fYvtku3Xatr2tb3S9i+z+32ydtu+3vYa26/a/uTurAwAYM+MFwqXSTrH9pO2v53dnpJ0nqRLx3ntrZIWjWi7QtLjEbFQ0uPZc0k6QdLC7LZEEteDBoACjDnQHBEfSDrS9mckHZI1PxART4y34Ih42vaCEc0nSTome7xC0pOS/jlrvy0iQtJztve2PT8i3mt0RQAAe67R6ymskrSqCe83r2ZD/76kednj/SS9WzPfuqxtp1CwvUTDvQkdeOCBTSgJAFBV2K+Ss15BjDvjzq9bHhG9EdHb09OTQ2UAMH21OhQ+sD1fkrL76qkz1ks6oGa+/bM2AEALtToU7pPUlz3uk3RvTfvi7CikwyV9yHgCALReo+c+2mW279DwoPJc2+skfV3SNyXdZfs8SWs1fBpuafg8SidKWiOpLOmcvOoCAIwut1CIiDNGmXRcnXlD0kV51QIAaAynvwYAJIQCACAhFAAACaEAAEgIBQBAQigAABJCAQCQEAoAgIRQAAAkhAIAICEUAAAJoQAASAgFAEBCKAAAEkIBAJAQCgCAhFAAACSEAgAgIRQAAAmhAABICAUAQEIoAAASQgEAkBAKAICEUAAAJIQCACAhFAAACaEAAEgIBQBAQigAABJCAQCQEAoAgIRQAAAkhAIAICEUAAAJoQAASAgFAEDSUcSb2n5H0gZJFUnbIqLX9r6S/lPSAknvSDo1Iv6viPoAYLoqsqfwmYg4NCJ6s+dXSHo8IhZKejx7DhRu25bNGhgYKLoMoCUm0u6jkyStyB6vkHRycaVguosIggDTUlGhEJIetf2S7SVZ27yIeC97/L6kefVeaHuJ7dW2V/f397eiVkxD5XJZi5euVGUoii4FaKlCxhQkHR0R623/gaSVtt+qnRgRYbvuX2NELJe0XJJ6e3v5i0VuOjpnEgqYdgrpKUTE+uy+JOkeSYdJ+sD2fEnK7ktF1AYA01nLQ8F2t+29qo8l/Y2k1yTdJ6kvm61P0r2trg0Aprsidh/Nk3SP7er7/ygiHrb9oqS7bJ8naa2kUwuoDQCmtZaHQkT8StLH67T/RtJxra4HALDdRDokFQBQMEIBAJAQCgCAhFAAACSEAgAgIRQAAAmhAABICAUAQEIoAOOonkY7gpPjYeojFIBxVLYO6h9WvKByuVx0KUDuCAVghHoX2GnvmllQNUBrEQrACFxgB9MZoQDU0dFJzwDTE6EAAEgIBQBAQigAABJCAQCQEAoAgIRQAAAkhAIAICEUAAAJoQAASAgFAEBCKAAAko6iCwAmiurZUUeeIRWYTggFIFMul3XK1fdqqDKk9q5ZO02PCG3cuFGS1N3dLdutLhHIHbuPgBodnTPrBoK0PTRO/c5DKpfLXJENUxKhAOyC2tAol8s67bqHuSIbphRCAdhFtT0ErruAqYZQAFT/EpyjqWwd1Je+/xQ9BExJDDRjWquGQalU0ld+8Izau2Y39LrRxh2AyY5QwJQQESqXy5o9e3Y6KmhkW0SoVCopIjRnzhx1d3enweOtmzapY/ZeYy6/kZ5EvTp2ZTpQNHYfYUqoN+g7sq1cLuuL1/1MZ13/SDqCSBoePG7rGntsoLJ1UBfc8owqQ8NHGtWGRO0YQ/U9+/v764ZIbU0cvYSJiFDAlNE+o2unjWx1ILi6AR4+emj4CKJdGUeQdtxlVA2Jwc2b1N/fr9Ouezj98K2jc+aYG/xqTRy9hImIUMCUUdk6qL4bn1SpVNLGjRt32CCXy2UtXrpyh2/6/f39O7TtqtqQaJ/RtcPyyuVyQ4PRox29RC8CRSEUMLW0tenMa+7XKVc/kDbItb2Equo3fXd07dHbVXcZ1S6v2tZes0uq+mvoaliNt8Hf3V4EYYI9RShgUtq4caNKpZKGhobSxraqrWum2jq370oarUfQjCOIKlsHdentq1UZirS8ytZB/eNtz2qoMiRpe6/klKvv1dk3PKryht9p8dJVu92LqC6z3safXVLYUxMuFGwvsv227TW2ryi6HrTGyG/S9aZt2LAh3UqlkvpuXKW1a9emjW3tRr/6zX3L4Oam9AjG0l5nkLqta2aqu1QqafHSlXJHVwqO2tCqrlu9da9d/9rp1Y3/wMDATtNGjqMMDQ3Re0DDJtQhqbbbJX1P0mclrZP0ou37IuKNZr4PhwWObbzPp7qBsT3mZ1j7bdZ2OolcvUNFq9/m3TZDK75yjObOnatyuZwGgvuWPZa+eUtSZcugOmbvpQtueUbtXbPUXud/cnUDXNRvCipbB3X2DY/WraEaWu1ds7bPM6NL3zurV93d3eru7pa0vZdx5rX3q31Gt/7jomPV09MjaXjjXx0raWvv1I+/emJ6TTUo+pY9pmV9R+riH72kH3z505o7d642bdqkWbNmpftqr2K0f5/RjDVf7SD+7Nmzd3jPqf5316zty2jLyXv7NaFCQdJhktZExK8kyfadkk6S1NRQKJfLOuVb/6UVF342/fFhu4GBAfUtXTnq51M9yubCFT8f8zMcGBjQF6+9X0OVUHtnl354yfHq7u7eafnV+dzeKVUGddZ3H9H3zz1aS256QkOV0PWLj6i7/MrgZrW1t6kyuGmH9qEtg4r2iqJ9547waNN25zWNLK8ta6+tsXZabXtl66DO/e5DmjF7jpaf/2lt27JZ/f39WnLTE4pKqKLhz+aHlxwvSdq2ZbMGBga0bctmdXR2pI3w4MCHOvOa+yVJ7Z1dacyj+rleuOLnWtp3ZLpfctMTcnvnqP8+oxlrvtp/05HvOdX/7hr9/HZ3OdX2n1x+ci6foydSl9L2KZIWRcT52fOzJf1VRFxcM88SSUuypx+T9HbLC919cyX9uugicsK6TU6s2+S0p+v2RxHRU2/CROspjCsilktaXnQdu8P26ojoLbqOPLBukxPrNjnluW4TbaB5vaQDap7vn7UBAFpgooXCi5IW2j7Idqek0yXdV3BNADBtTKjdRxGxzfbFkh6R1C7ploh4veCymmlS7vZqEOs2ObFuk1Nu6zahBpoBAMWaaLuPAAAFIhQAAAmh0GK2v2X7Lduv2r7H9t5F19Qstr9g+3XbQ7Yn/aGAU/mUK7ZvsV2y/VrRtTST7QNsr7L9RvZ/8dKia2oW2zNtv2D7v7N1+7c83odQaL2Vkg6JiL+U9D+Sriy4nmZ6TdLnJT1ddCF7quaUKydIOljSGbYPLraqprpV0qKii8jBNklfi4iDJR0u6aIp9O82KOnYiPi4pEMlLbJ9eLPfhFBosYh4NCK2ZU+f0/BvMaaEiHgzIibTL8zHkk65EhFbJFVPuTIlRMTTkn5bdB3NFhHvRcTL2eMNkt6UtF+xVTVHDKueDnhGdmv6kUKEQrHOlfRQ0UWgrv0kvVvzfJ2myMZlurC9QNInJD1fcClNY7vd9iuSSpJWRkTT121C/U5hqrD9mKQ/rDPpqoi4N5vnKg13dW9vZW17qpF1A4pme46kn0q6LCI+KrqeZomIiqRDs7HIe2wfEhFNHRciFHIQEX891nTbX5L0t5KOi0n2Q5Hx1m0K4ZQrk5TtGRoOhNsj4u6i68lDRPzO9ioNjws1NRTYfdRithdJ+idJfx8RXB5r4uKUK5OQhy8wcLOkNyPimqLraSbbPdWjFW3P0vB1Z95q9vsQCq13g6S9JK20/YrtG4suqFlsf872OklHSHrA9iNF17S7soMBqqdceVPSXVPplCu275D0rKSP2V5n+7yia2qSoySdLenY7O/rFdsnFl1Uk8yXtMr2qxr+0rIyIn7W7DfhNBcAgISeAgAgIRQAAAmhAABICAUAQEIoAAASQgEAkBAKAIDk/wEk1Qnma8UBrgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Here are the weights kernel histogram\n",
    "sns.histplot(data=data_reports['test_hexos']['facts0_and_kernel'].flatten())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 7.02416],\n",
       "       [ 7.82561],\n",
       "       [ 7.94817],\n",
       "       [ 7.95645],\n",
       "       [10.73285],\n",
       "       [ 9.43431],\n",
       "       [-6.17754],\n",
       "       [10.20865],\n",
       "       [-5.96961],\n",
       "       [-6.48949]], dtype=float32)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# These are the reported prediction values after the training is complete\n",
    "data_reports['test_hexos']['prediction_label'][:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 7.02416],\n",
       "       [ 7.87824],\n",
       "       [ 7.94817],\n",
       "       [ 7.9571 ],\n",
       "       [10.73285],\n",
       "       [ 9.43431],\n",
       "       [-6.17754],\n",
       "       [10.20864],\n",
       "       [-5.96961],\n",
       "       [-6.48949]], dtype=float32)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Which should be very similar to the dnf_model, note they can vary since the object selection is stochastic\n",
    "dnf_model.predict(datasets['test_hexos'], verbose=0)['label'][:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:ylabel='Count'>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAD4CAYAAAAdIcpQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Z1A+gAAAACXBIWXMAAAsTAAALEwEAmpwYAAAVmElEQVR4nO3df7DldX3f8edLEEyrCYtst7Dsumuz1mAzQecGiaaJPyIgM3WxJbhOoxuDXWMgEyc2U4h/aLVOTScJrY1FiW7F1IoEdVwTErr8ME4mgiwpAgtBrijZXZHdCGJSp8TFd/84n5t8e7l3v+cu53vuXfb5mDlzvuf9/XHe93vOPa/z/XHOSVUhSdKhPG25G5AkrXyGhSSpl2EhSeplWEiSehkWkqRexy53A0M46aSTasOGDcvdhiQdUW677ba/qqrVC417SobFhg0b2LVr13K3IUlHlCQPLDbO3VCSpF6DhUWSZyT5UpIvJ9md5N+3+sYktySZTfLJJMe1+vHt9mwbv6GzrEtb/d4kZw/VsyRpYUNuWTwGvKKqfgw4HTgnyZnAbwCXVdUPA48AF7bpLwQeafXL2nQkOQ3YArwAOAf4b0mOGbBvSdI8g4VFjfxNu/n0dingFcA1rX4lcF4b3txu08a/Mkla/aqqeqyqvgbMAmcM1bck6YkGPWaR5JgktwP7gZ3AV4FvV9XBNsleYG0bXgvsAWjjHwWe3a0vME/3vrYl2ZVk14EDBwb4ayTp6DVoWFTV41V1OnAqo62B5w94X1dU1UxVzaxeveCZX5KkwzSVs6Gq6tvATcBPACckmTtl91RgXxveB6wDaON/CPhWt77APJKkKRjybKjVSU5owz8AvAq4h1FonN8m2wp8tg3vaLdp42+s0fen7wC2tLOlNgKbgC8N1bck6YmG/FDeycCV7cylpwFXV9UfJLkbuCrJfwD+N/CRNv1HgN9LMgs8zOgMKKpqd5KrgbuBg8BFVfX4gH1LkubJU/HHj2ZmZspPcEtaydauW8839u7pn3CJTjl1Hfv2/OVhzZvktqqaWWjcU/LrPiRppfvG3j287kN/NvHlfvItL5n4MsGv+5AkjcGwkCT1MiwkSb0MC0lSL8NCktTLsJAk9TIsJEm9DAtJUi/DQpLUy7BYwNp160ky0cvadeuX+8+SpMPm130sYIiP4Q/1EXxJmga3LCRJvQwLSVIvw0KS1MuwkCT1MiwkSb0MC0lSL8NCktTLsJAk9TIsJEm9DAtJUi/DQpLUy7CQJPUyLCRJvQwLSVKvwcIiybokNyW5O8nuJL/S6u9Ksi/J7e1ybmeeS5PMJrk3ydmd+jmtNpvkkqF6liQtbMjfszgIvL2q/jzJs4Dbkuxs4y6rqt/sTpzkNGAL8ALgFOD6JM9roz8AvArYC9yaZEdV3T1g75KkjsHCoqoeBB5sw3+d5B5g7SFm2QxcVVWPAV9LMguc0cbNVtX9AEmuatMaFpI0JVM5ZpFkA/BC4JZWujjJHUm2J1nVamuBPZ3Z9rbaYvX597Etya4kuw4cODDpP0GSjmqDh0WSZwKfAt5WVd8BLgf+CXA6oy2P35rE/VTVFVU1U1Uzq1evnsQiJUnNoL/BneTpjILi41X1aYCqeqgz/neBP2g39wHrOrOf2mocoi5JmoIhz4YK8BHgnqr67U795M5krwXuasM7gC1Jjk+yEdgEfAm4FdiUZGOS4xgdBN8xVN+SpCcacsvipcAbgDuT3N5qvw68PsnpQAFfB94CUFW7k1zN6MD1QeCiqnocIMnFwHXAMcD2qto9YN+SpHmGPBvqT4EsMOraQ8zzXuC9C9SvPdR8kqRh+QluSVIvw0KS1MuwkCT1MiwkSb0MC0lSL8NCktTLsJAk9TIsJEm9DAtJUi/DQpLUy7CQJPUyLCRJvQwLSVIvw0KS1MuwkCT1MiwkSb0MC0lSL8NCktTLsJAk9TIsJEm9DAtJUi/DQpLUy7CQJPUyLCRJvQwLSVIvw0KS1MuwkCT1GiwskqxLclOSu5PsTvIrrX5ikp1J7mvXq1o9Sd6fZDbJHUle1FnW1jb9fUm2DtWzJGlhQ25ZHATeXlWnAWcCFyU5DbgEuKGqNgE3tNsArwY2tcs24HIYhQvwTuDFwBnAO+cCRpI0HYOFRVU9WFV/3ob/GrgHWAtsBq5sk10JnNeGNwMfq5GbgROSnAycDeysqoer6hFgJ3DOUH1Lkp5oKscskmwAXgjcAqypqgfbqG8Ca9rwWmBPZ7a9rbZYff59bEuyK8muAwcOTPYPkKSj3OBhkeSZwKeAt1XVd7rjqqqAmsT9VNUVVTVTVTOrV6+exCIlSc2gYZHk6YyC4uNV9elWfqjtXqJd72/1fcC6zuynttpidUnSlAx5NlSAjwD3VNVvd0btAObOaNoKfLZTf2M7K+pM4NG2u+o64Kwkq9qB7bNaTZI0JccOuOyXAm8A7kxye6v9OvA+4OokFwIPABe0cdcC5wKzwHeBNwFU1cNJ3gPc2qZ7d1U9PGDfkqR5BguLqvpTIIuMfuUC0xdw0SLL2g5sn1x3kqSl8BPckqRehoUkqZdhIUnqZVhIknoZFpKkXoaFJKmXYSFJ6mVYSJJ6GRaSpF6GhSSpl2EhSeplWEiSehkWkqRehoUkqddYYZHkpePUJElPTeNuWfzXMWuSpKegQ/74UZKfAF4CrE7yq51RPwgcM2RjkqSVo++X8o4Dntmme1an/h3g/KGakiStLIcMi6r6E+BPkny0qh6YUk+SpBVm3N/gPj7JFcCG7jxV9YohmpIkrSzjhsXvAx8EPgw8Plw7kqSVaNywOFhVlw/aiSRpxRr31NnPJfmlJCcnOXHuMmhnkqQVY9wti63t+tc6tQKeO9l2JEkr0VhhUVUbh25EkrRyjRUWSd64UL2qPjbZdiRJK9G4xyx+vHP558C7gNccaoYk25PsT3JXp/auJPuS3N4u53bGXZpkNsm9Sc7u1M9ptdkklyzhb5MkTci4u6F+uXs7yQnAVT2zfRT4HWD+1sdlVfWb85Z3GrAFeAFwCnB9kue10R8AXgXsBW5NsqOq7h6nb0nSZIx7gHu+/wMc8jhGVX0hyYYxl7cZuKqqHgO+lmQWOKONm62q+wGSXNWmNSwkaYrGPWbxOUZnP8HoCwR/BLj6MO/z4nYMZBfw9qp6BFgL3NyZZm+rAeyZV3/xIj1uA7YBrF+//jBbkyQtZNwti+5uo4PAA1W19zDu73LgPYyC5z3AbwG/cBjLeYKqugK4AmBmZqZ6JpckLcFYB7jbFwr+BaNvnl0F/O3h3FlVPVRVj1fV94Hf5e93Ne0D1nUmPbXVFqtLkqZo3F/KuwD4EvCzwAXALUmW/BXlSU7u3HwtMHem1A5gS5Ljk2wENrX7uxXYlGRjkuMYHQTfsdT7lSQ9OePuhnoH8ONVtR8gyWrgeuCaxWZI8gngZcBJSfYC7wReluR0Rruhvg68BaCqdie5mtGB64PARVX1eFvOxcB1jI6VbK+q3Uv7EyVJT9a4YfG0uaBovkXPVklVvX6B8kcOMf17gfcuUL8WuHbMPiVJAxg3LP44yXXAJ9rt1+ELuCQdNfp+g/uHgTVV9WtJ/iXwk23UF4GPD92cJGll6Nuy+M/ApQBV9Wng0wBJfrSN+xcD9iZJWiH6zoZaU1V3zi+22oZBOpIkrTh9YXHCIcb9wAT7kCStYH1hsSvJv5lfTPJm4LZhWpIkrTR9xyzeBnwmyb/m78NhBjiO0YfqJElHgUOGRVU9BLwkycuBf9bKf1hVNw7emSRpxRj39yxuAm4auBdJ0go17i/lSZKOYoaFJKmXYSFJ6mVYSJJ6GRaSpF6GhSSpl2EhSeplWEiSehkWkqRehoUkqZdhIUnqZVhIknoZFpKkXoaFJKmXYSFJ6mVYSJJ6GRaSpF6DhUWS7Un2J7mrUzsxyc4k97XrVa2eJO9PMpvkjiQv6syztU1/X5KtQ/UrSVrckFsWHwXOmVe7BLihqjYBN7TbAK8GNrXLNuByGIUL8E7gxcAZwDvnAkaSND2DhUVVfQF4eF55M3BlG74SOK9T/1iN3AyckORk4GxgZ1U9XFWPADt5YgBJkgY27WMWa6rqwTb8TWBNG14L7OlMt7fVFqs/QZJtSXYl2XXgwIHJdi1JR7llO8BdVQXUBJd3RVXNVNXM6tWrJ7VYSRLTD4uH2u4l2vX+Vt8HrOtMd2qrLVaXJE3RtMNiBzB3RtNW4LOd+hvbWVFnAo+23VXXAWclWdUObJ/VapKkKTp2qAUn+QTwMuCkJHsZndX0PuDqJBcCDwAXtMmvBc4FZoHvAm8CqKqHk7wHuLVN9+6qmn/QXJI0sMHCoqpev8ioVy4wbQEXLbKc7cD2CbYmSVoiP8EtSeplWEiSehkWkqRehoUkqZdhIUnqZVhIknoZFpKkXoaFJKmXYSFJ6mVYSJJ6GRaSpF6GhSSpl2EhSeplWEiSehkWkqRehoUkqZdhIUnqZVhIknoZFpKkXoaFJKmXYSFJ6mVYSJJ6GRaSpF6GhSSpl2EhSeplWEiSei1LWCT5epI7k9yeZFernZhkZ5L72vWqVk+S9yeZTXJHkhctR8+SdDRbzi2Ll1fV6VU1025fAtxQVZuAG9ptgFcDm9plG3D51DuVpKPcStoNtRm4sg1fCZzXqX+sRm4GTkhy8jL0J0lHreUKiwL+V5LbkmxrtTVV9WAb/iawpg2vBfZ05t3bapKkKTl2me73J6tqX5J/BOxM8hfdkVVVSWopC2yhsw1g/fr1k+tUkrQ8WxZVta9d7wc+A5wBPDS3e6ld72+T7wPWdWY/tdXmL/OKqpqpqpnVq1cP2b4kHXWmHhZJ/mGSZ80NA2cBdwE7gK1tsq3AZ9vwDuCN7ayoM4FHO7urJElTsBy7odYAn0kyd///s6r+OMmtwNVJLgQeAC5o018LnAvMAt8F3jT9liXp6Db1sKiq+4EfW6D+LeCVC9QLuGgKrUmSFrGSTp2VJK1QhoUkqZdhIUnqZVhIknoZFpKkXoaFJKmXYSFJ6mVYSJJ6GRaSpF6GhSSpl2EhSeplWEiSehkWkqRehoUkqZdhIUnqZVhIknoZFpKkXoaFJKmXYSFJ6mVYSJJ6GRaSpF6GhSSpl2EhSeplWEiSehkWkqRehoUkqZdhIWlRa9etJ8nEL2vXrV/uP01LdOxyNzCuJOcA/wU4BvhwVb1vmVtamqcdS5KJL/aUU9exb89fTny5Gr1QfmPvnokv95inH8/j33vsiFnu6z70ZxNf5iff8pKJLxOOvMfsSHJEhEWSY4APAK8C9gK3JtlRVXcvb2dL8P2Dw/zTvfWnBgmhI+kFbch/5KFeKI+U5Q71oj7Umyc4ch6zwdbtQI6IsADOAGar6n6AJFcBm4EjJyyGMlQIHWEvaEfSu18x6PNWw0hVLXcPvZKcD5xTVW9ut98AvLiqLu5Msw3Y1m7+U+DeJ3GXJwF/9STmH4p9LY19LY19Lc1Tsa/nVNXqhUYcKVsWvarqCuCKSSwrya6qmpnEsibJvpbGvpbGvpbmaOvrSDkbah+wrnP71FaTJE3BkRIWtwKbkmxMchywBdixzD1J0lHjiNgNVVUHk1wMXMfo1NntVbV7wLucyO6sAdjX0tjX0tjX0hxVfR0RB7glScvrSNkNJUlaRoaFJKnXURsWSX42ye4k30+y6GlmSc5Jcm+S2SSXdOobk9zS6p9sB94n0deJSXYmua9dr1pgmpcnub1z+b9JzmvjPprka51xp0+rrzbd45373tGpL+f6Oj3JF9vjfUeS13XGTWx9LfZc6Yw/vv3ts21dbOiMu7TV701y9uH2cJh9/WqSu9u6uSHJczrjFnw8p9jbzyc50OnhzZ1xW9vjfl+SrVPs6bJOP19J8u3OuMHWV5LtSfYnuWuR8Uny/tb3HUle1Bn35NdVVR2VF+BHGH147/PAzCLTHAN8FXgucBzwZeC0Nu5qYEsb/iDw1gn19Z+AS9rwJcBv9Ex/IvAw8A/a7Y8C5w+wvsbqC/ibRerLtr6A5wGb2vApwIPACZNcX4d6rnSm+SXgg214C/DJNnxam/54YGNbzjETWj/j9PXyzvPnrXN9HerxnGJvPw/8zgLzngjc365XteFV0+hp3vS/zOiEm2msr58CXgTctcj4c4E/AgKcCdwyyXV11G5ZVNU9VdX3Ke+/+5qRqvpb4Cpgc5IArwCuadNdCZw3odY2t+WNu9zzgT+qqu9O6P4Xs9S+/s5yr6+q+kpV3deGvwHsBxb8lOqTsOBz5RC9XgO8sq2bzcBVVfVYVX0NmG3Lm0pfVXVT5/lzM6PPMU3DOOtsMWcDO6vq4ap6BNgJnLMMPb0e+MQE7rdXVX2B0RvDxWwGPlYjNwMnJDmZCa2rozYsxrQW6H6F5d5Wezbw7ao6OK8+CWuq6sE2/E1gTc/0W3jik/W9bTP0siTHT7mvZyTZleTmuV1jrKD1leQMRu8Yv9opT2J9LfZcWXCati4eZbRuxpn3cC112Rcyenc6Z6HHc1LG7e1ftcfnmiRzH84dap2Nvdy2u24jcGOnPOT66rNY7xNZV0fE5ywOV5LrgX+8wKh3VNVnp93PnEP11b1RVZVk0XOb27uGH2X0+ZM5lzJ60TyO0fnW/w549xT7ek5V7UvyXODGJHcyelE8bBNeX78HbK2q77fyYa+vp5okPwfMAD/dKT/h8ayqry68hEF8DvhEVT2W5C2MtsxeMcX7P5QtwDVV9XinttzrazBP6bCoqp95kotY7GtGvsVoE+/Y9g5xSV8/cqi+kjyU5OSqerC9uO0/xKIuAD5TVd/rLHvuXfZjSf478G+n2VdV7WvX9yf5PPBC4FMs8/pK8oPAHzJ6o3BzZ9mHvb7mGecraeam2ZvkWOCHGD2Xhvw6m7GWneRnGIXvT1fV333f+yKP56Re/Hp7q6pvdW5+mNExqrl5XzZv3s9Po6eOLcBF3cLA66vPYr1PZF25G+rQFvyakRodNbqJ0fECgK3ApLZUdrTljbPcJ+wvbS+Yc8cJzgMWPHNiiL6SrJrbjZPkJOClwN3Lvb7aY/cZRvtzr5k3blLra5yvpOn2ej5wY1s3O4AtGZ0ttRHYBHzpMPtYcl9JXgh8CHhNVe3v1Bd8PCfU17i9ndy5+RrgnjZ8HXBW63EVcBb//xb2YD21vp7P6GDxFzu1oddXnx3AG9tZUWcCj7Y3Q5NZV0MduV/pF+C1jPbdPQY8BFzX6qcA13amOxf4CqN3B+/o1J/L6B96Fvh94PgJ9fVs4AbgPuB64MRWn2H0C4Fz021g9I7hafPmvxG4k9GL3v8AnjmtvoCXtPv+cru+cCWsL+DngO8Bt3cup096fS30XGG0S+s1bfgZ7W+fbeviuZ1539Hmuxd49YSf6319Xd/+B+bWzY6+x3OKvf1HYHfr4Sbg+Z15f6Gty1ngTdPqqd1+F/C+efMNur4YvTF8sD2X9zI6vvSLwC+28WH0I3Ffbfc/05n3Sa8rv+5DktTL3VCSpF6GhSSpl2EhSeplWEiSehkWkqRehoUkqZdhIUnq9f8A035Qv8M7iRoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# These are the relations computed between the objects\n",
    "sns.histplot(data=data_reports['test_hexos']['report_layer_binary'].flatten())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:ylabel='Count'>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAD4CAYAAAAD6PrjAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Z1A+gAAAACXBIWXMAAAsTAAALEwEAmpwYAAAReUlEQVR4nO3df7BcZX3H8fcXUrCtWhJJ0xgSA21aTes0MldK0akgjgIzNdgihBkl0tigoqNj6wjlD53OOKWdWlttq6RKgdbyQ5QhjiiFADodBQ0O8lMkoDQJIYmoaOsUDXz7xz738Zjszd2bu2fPzb3v18zOnn3Or+99du9+9vzYs5GZSJIEcEjXBUiSZg5DQZJUGQqSpMpQkCRVhoIkqZrXdQHTceSRR+by5cu7LkOSDip33nnndzNzYb9xB3UoLF++nM2bN3ddhiQdVCLi0YnGuftIklQZCpKkylCQJFWGgiSpMhQkSZWhIEmqDAVJUmUoSJIqQ0GSVM3ZUFiydBkR0cltydJlXf/5ktTXQX2Zi+l4bNtWzrrky52s++rzTuhkvZI0mTm7pSBJ2pehIEmqDAVJUmUoSJIqQ0GSVBkKkqTKUJAkVYaCJKkyFCRJlaEgSaoMBUlSZShIkipDQZJUGQqSpMpQkCRVhoIkqTIUJEmVoSBJqloLhYhYGhG3RsT9EXFfRLyztC+IiJsi4qFyP7+0R0R8OCK2RMTdEXFsW7VJkvprc0thD/BnmbkSOB44PyJWAhcAmzJzBbCpPAY4FVhRbuuBj7ZYmySpj9ZCITN3ZObXy/CPgAeAJcBq4PIy2eXA6WV4NXBF9twOHBERi9uqT5K0r5EcU4iI5cBLgDuARZm5o4x6HFhUhpcAWxuzbSttkqQRaT0UIuLZwKeBd2XmD5vjMjOBnOLy1kfE5ojYvHv37iFWKklqNRQi4hfoBcInM/MzpXnn+G6hcr+rtG8HljZmP6q0/ZzM3JCZY5k5tnDhwvaKl6Q5qM2zjwL4BPBAZv5dY9RGYG0ZXgtc32g/p5yFdDzwZGM3kyRpBOa1uOyXAW8E7omIu0rbXwAXA9dExDrgUeDMMu4G4DRgC/Bj4NwWa5Mk9dFaKGTmfwExweiT+0yfwPlt1SNJmpzfaJYkVYaCJKkyFCRJlaEgSaoMBUlSZShIkipDQZJUGQqSpMpQkCRVhoIkqTIUJEmVoSBJqgwFSVJlKEiSKkNBklQZCpKkylCQJFWGgiSpMhQkSZWhIEmqDAVJUmUoSJIqQ0GSVBkKkqTKUJAkVYaCJKkyFCRJlaEgSaoMBUlSZShIkipDQZJUGQqSpMpQkCRVhoIkqTIUJOkALVm6jIjo5LZk6bJW/qZ5rSxVkuaAx7Zt5axLvtzJuq8+74RWluuWgiSpai0UIuLSiNgVEfc22t4fEdsj4q5yO60x7sKI2BIRD0bEa9qqS5I0sTa3FC4DTunT/qHMXFVuNwBExEpgDfDbZZ5/johDW6xNktRHa6GQmV8Cvjfg5KuBqzLzqcz8NrAFOK6t2iRJ/XVxTOHtEXF32b00v7QtAbY2ptlW2vYREesjYnNEbN69e3fbtUrSnDLqUPgo8OvAKmAH8MGpLiAzN2TmWGaOLVy4cMjlSdLcNtJQyMydmfl0Zj4D/As/20W0HVjamPSo0iZJGqGRhkJELG48fB0wfmbSRmBNRBweEUcDK4CvjrI2SVKLX16LiCuBE4EjI2Ib8D7gxIhYBSTwHeA8gMy8LyKuAe4H9gDnZ+bTbdUmSeqvtVDIzLP7NH9iP9N/APhAW/VIkibnN5olSZWhIEmqDAVJUmUoSJIqQ0GSVBkKkqTKUJAkVYaCJKkyFCRJlaEgSaoMBUlSNVAoRMTLBmmTJB3cBt1S+MiAbZKkg9h+r5IaEb8PnAAsjIh3N0Y9Fzi0zcIkSaM32aWzDwOeXaZ7TqP9h8AZbRUlSerGfkMhM78IfDEiLsvMR0dUkySpI4P+yM7hEbEBWN6cJzNf2UZRkqRuDBoKnwI+Bnwc8GcyJWmWGjQU9mTmR1utRJLUuUFPSf1sRLwtIhZHxILxW6uVSZJGbtAthbXl/j2NtgSOGW45kqQuDRQKmXl024VIkro3UChExDn92jPziuGWI0nq0qC7j17aGH4WcDLwdcBQkKRZZNDdR+9oPo6II4Cr2ihIktSdA7109v8CHmeQpFlm0GMKn6V3thH0LoT3IuCatoqSJHVj0GMKf9sY3gM8mpnbWqhHktShgXYflQvjfZPelVLnAz9psyhJUjcG/eW1M4GvAq8HzgTuiAgvnS1Js8ygu48uAl6ambsAImIhcDNwbVuFSZJGb9Czjw4ZD4TiiSnMK0k6SAy6pfCFiLgRuLI8Pgu4oZ2SJEldmew3mn8DWJSZ74mIPwJeXkZ9Bfhk28VJkkZrsi2FvwcuBMjMzwCfAYiIF5dxf9hibZKkEZvsuMCizLxn78bStryViiRJnZksFI7Yz7hfHGIdkqQZYLJQ2BwRf7p3Y0S8GbhzfzNGxKURsSsi7m20LYiImyLioXI/v7RHRHw4IrZExN0RceyB/DGSpOmZLBTeBZwbEbdFxAfL7YvAOuCdk8x7GXDKXm0XAJsycwWwqTwGOBVYUW7rAX8PWpI6sN8DzZm5EzghIk4Cfqc0fy4zb5lswZn5pYhYvlfzauDEMnw5cBvw3tJ+RWYmcHtEHBERizNzx6B/iCRp+gb9PYVbgVuHsL5FjTf6x4FFZXgJsLUx3bbStk8oRMR6elsTLFu2bAglSZLGdfat5LJVkJNOuO98GzJzLDPHFi5c2EJlkjR3jToUdkbEYoByP37pjO3A0sZ0R5U2SdIIjToUNgJry/Ba4PpG+znlLKTjgSc9niBJozfotY+mLCKupHdQ+ciI2Aa8D7gYuCYi1gGP0rsMN/Suo3QasAX4MXBuW3VJkibWWihk5tkTjDq5z7QJnN9WLZKkwXj5a0lSZShIkipDQZJUGQqSpMpQkCRVhoIkqTIUJEmVoSBJqgwFSVJlKEiSKkNBklQZCpKkylCQJFWGgiSpMhQkSZWhIEmqDAVJUmUoSJIqQ0GSVBkKkqTKUJAkVYaCJKkyFCRJlaEgSaoMBUlSZShIkipDQZJUGQqSpMpQkCRVhoIkqTIUJEmVoSBJqgwFSVJlKEiSKkNBklQZCpKkylCQJFXzulhpRHwH+BHwNLAnM8ciYgFwNbAc+A5wZmZ+v4v6JGmu6nJL4aTMXJWZY+XxBcCmzFwBbCqPJUkjNJN2H60GLi/DlwOnd1eKJM1NXYVCAv8ZEXdGxPrStigzd5Thx4FF/WaMiPURsTkiNu/evXsUtUrSnNHJMQXg5Zm5PSJ+FbgpIr7ZHJmZGRHZb8bM3ABsABgbG+s7jSTpwHSypZCZ28v9LuA64DhgZ0QsBij3u7qoTZLmspGHQkT8ckQ8Z3wYeDVwL7ARWFsmWwtcP+raJGmu62L30SLguogYX/9/ZOYXIuJrwDURsQ54FDizg9okaU4beShk5iPA7/ZpfwI4edT1SJJ+ZiadkipJ6pihIEmqDAVJUmUoSJIqQ0GSVBkKkqTKUJAkVYaCJKkyFCRJlaEgSaoMBUlSZShIkipDQWrJkqXLiIiR35YsXdb1n66DWFe/vCbNeo9t28pZl3x55Ou9+rwTRr5OzR5uKUiSKkNBklQZCpKkymMK0mxzyDzKz92O1POPWsr2rf898vVquAwFabZ5Zo8HuHXA3H0kSaoMBUlS5e4jScPR0bEM8HjGMBkKkoajo2MZ4PGMYXL3kUbCSz5IBwe3FDQSXvJBOji4pSBJqgwFSVJlKEiSKo8pdMHLEEjD1eHpsLONodAFL0MgDZf/U0Pj7iNJUmUoSJIqQ0GSVBkKkqTKA82a3TwrRZoSQ0Gzmxdpk6bEUJhL/NQsaRIzLhQi4hTgH4BDgY9n5sUdlzR7+KlZ0iRm1IHmiDgU+CfgVGAlcHZErOy2KkmaO2ZUKADHAVsy85HM/AlwFbC645okac6IzOy6hioizgBOycw3l8dvBH4vM9/emGY9sL48/C3gwQNc3ZHAd6dRbptmam3WNTXWNTUztS6YubUdaF0vyMyF/UbMuGMKk8nMDcCG6S4nIjZn5tgQShq6mVqbdU2NdU3NTK0LZm5tbdQ103YfbQeWNh4fVdokSSMw00Lha8CKiDg6Ig4D1gAbO65JkuaMGbX7KDP3RMTbgRvpnZJ6aWbe19Lqpr0LqkUztTbrmhrrmpqZWhfM3NqGXteMOtAsSerWTNt9JEnqkKEgSapmdShExOsj4r6IeCYiJjxtKyJOiYgHI2JLRFzQaD86Iu4o7VeXg9/DqGtBRNwUEQ+V+/l9pjkpIu5q3P4vIk4v4y6LiG83xq0aRl2D1lame7qx/o2N9i77bFVEfKU853dHxFmNcUPts4leM43xh5e/f0vpj+WNcReW9gcj4jXTqeMA6np3RNxf+mdTRLygMa7vczqiut4UEbsb639zY9za8rw/FBFrR1zXhxo1fSsiftAY12Z/XRoRuyLi3gnGR0R8uNR9d0Qc2xg3vf7KzFl7A15E7wtutwFjE0xzKPAwcAxwGPANYGUZdw2wpgx/DHjrkOr6G+CCMnwB8NeTTL8A+B7wS+XxZcAZLfXZQLUB/zNBe2d9BvwmsKIMPx/YARwx7D7b32umMc3bgI+V4TXA1WV4ZZn+cODospxDR1jXSY3X0VvH69rfczqiut4E/GOfeRcAj5T7+WV4/qjq2mv6d9A7+aXV/irL/gPgWODeCcafBnweCOB44I5h9des3lLIzAcyc7JvPPe9tEZEBPBK4Noy3eXA6UMqbXVZ3qDLPQP4fGb+eEjr35+p1lZ13WeZ+a3MfKgMPwbsAvp+a3OaBrkcS7Pea4GTS/+sBq7KzKcy89vAlrK8kdSVmbc2Xke30/suUNumc/ma1wA3Zeb3MvP7wE3AKR3VdTZw5ZDWvV+Z+SV6HwQnshq4IntuB46IiMUMob9mdSgMaAmwtfF4W2l7HvCDzNyzV/swLMrMHWX4cWDRJNOvYd8X4wfKZuOHIuLwIdU1ldqeFRGbI+L28d1azKA+i4jj6H36e7jRPKw+m+g103ea0h9P0uufQeZts66mdfQ+bY7r95yOsq4/Ls/PtREx/iXWGdFfZTfb0cAtjea2+msQE9U+7f6aUd9TOBARcTPwa31GXZSZ14+6nnH7q6v5IDMzIiY8L7ik/4vpfXdj3IX03hgPo3ee8nuBvxxxbS/IzO0RcQxwS0TcQ++N74ANuc/+DVibmc+U5mn12WwTEW8AxoBXNJr3eU4z8+H+Sxi6zwJXZuZTEXEeva2sV45o3YNYA1ybmU832rrsr9Yc9KGQma+a5iImurTGE/Q2yeaVT3pTuuTG/uqKiJ0RsTgzd5Q3sF37WdSZwHWZ+dPGssc/MT8VEf8K/PmgdQ2rtszcXu4fiYjbgJcAn6bjPouI5wKfo/eh4PbGsqfVZ3sZ5HIs49Nsi4h5wK/Qe021eSmXgZYdEa+iF7SvyMynxtsneE6H8SY3aV2Z+UTj4cfpHUMan/fEvea9bQg1DVRXwxrg/GZDi/01iIlqn3Z/uftogktrZO+oza309ucDrAWGteWxsSxvkOXusx+zvCmO78M/Heh7hkJbtUXE/PHdLxFxJPAy4P6u+6w8f9fR29d67V7jhtlng1yOpVnvGcAtpX82Amuid3bS0cAK4KvTqGVKdUXES4BLgNdm5q5Ge9/ndIR1LW48fC3wQBm+EXh1qW8+8Gp+fqu51bpKbS+kd9D2K422NvtrEBuBc8pZSMcDT5YPPtPvr7aOns+EG/A6evvUngJ2AjeW9ucDNzSmOw34Fr2Uv6jRfgy9f9gtwKeAw4dU1/OATcBDwM3AgtI+Ru/X5sanW04v+Q/Za/5bgHvovbH9O/DsIfbZpLUBJ5T1f6Pcr5sJfQa8AfgpcFfjtqqNPuv3mqG3O+q1ZfhZ5e/fUvrjmMa8F5X5HgROHfJrfrK6bi7/C+P9s3Gy53REdf0VcF9Z/63ACxvz/knpxy3AuaOsqzx+P3DxXvO13V9X0jt77qf03sPWAW8B3lLGB70fJHu4rH+sMe+0+svLXEiSKncfSZIqQ0GSVBkKkqTKUJAkVYaCJKkyFCRJlaEgSar+H/Itgr+1SflKAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# These are the outputs of the conjunctive SL post existential reduction using the max operator\n",
    "# the max(1) reduces the permutations\n",
    "sns.histplot(data=data_reports['test_hexos']['facts0_conjuncts'].max(1).flatten())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1.55504, -1.7244 ,  7.44871, -2.22057, -2.27163,  2.75774, -1.45883,  1.74555]], dtype=float32)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# This is the actual weight used in the disjunctive SL, or_kernel\n",
    "data_reports['test_hexos']['facts0_or_kernel']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1, 8, 120)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Similarly you can find the conjunctive SL kernel here\n",
    "data_reports['test_hexos']['facts0_and_kernel'].shape"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Muli Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}