fstorebasics.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>credit_card_number</th>\n",
       "      <th>trans_datetime</th>\n",
       "      <th>amount</th>\n",
       "      <th>location</th>\n",
       "      <th>fraud</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1111 2222 3333 4444</td>\n",
       "      <td>2022-01-01 08:44:00</td>\n",
       "      <td>142.34</td>\n",
       "      <td>Sao Paolo</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1111 2222 3333 4444</td>\n",
       "      <td>2022-01-02 19:44:00</td>\n",
       "      <td>12.34</td>\n",
       "      <td>Rio De Janeiro</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1111 2222 3333 4444</td>\n",
       "      <td>2022-01-02 20:44:00</td>\n",
       "      <td>66.29</td>\n",
       "      <td>Stockholm</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1111 2222 3333 4444</td>\n",
       "      <td>2022-01-02 20:55:00</td>\n",
       "      <td>112.33</td>\n",
       "      <td>Stockholm</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    credit_card_number      trans_datetime  amount        location  fraud\n",
       "0  1111 2222 3333 4444 2022-01-01 08:44:00  142.34       Sao Paolo  False\n",
       "1  1111 2222 3333 4444 2022-01-02 19:44:00   12.34  Rio De Janeiro  False\n",
       "2  1111 2222 3333 4444 2022-01-02 20:44:00   66.29       Stockholm   True\n",
       "3  1111 2222 3333 4444 2022-01-02 20:55:00  112.33       Stockholm   True"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "\n",
    "data = { \n",
    "    'credit_card_number': ['1111 2222 3333 4444', '1111 2222 3333 4444','1111 2222 3333 4444',\n",
    "                           '1111 2222 3333 4444'],\n",
    "    'trans_datetime': ['2022-01-01 08:44', '2022-01-02 19:44', '2022-01-02 20:44', '2022-01-02 20:55'],\n",
    "    'amount': [142.34, 12.34, 66.29, 112.33],\n",
    "    'location': ['Sao Paolo', 'Rio De Janeiro', 'Stockholm', 'Stockholm'],\n",
    "    'fraud': [False, False, True, True] \n",
    "}\n",
    "\n",
    "df = pd.DataFrame.from_dict(data)\n",
    "df['trans_datetime']= pd.to_datetime(df['trans_datetime'])\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2025-03-05 02:40:17,168 INFO: Initializing external client\n",
      "2025-03-05 02:40:17,169 INFO: Base URL: https://c.app.hopsworks.ai:443\n",
      "2025-03-05 02:40:17,798 INFO: Python Engine initialized.\n",
      "\n",
      "Logged in to project, explore it here https://c.app.hopsworks.ai:443/p/1214673\n"
     ]
    }
   ],
   "source": [
    "import hopsworks\n",
    "proj = hopsworks.login(api_key_value=os.getenv(\"HOPSWORKS_PYTHON_CLIKEY\"))\n",
    "fs = proj.get_feature_store()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "fg = fs.get_or_create_feature_group(\n",
    "     name=\"credit_card_transactions\",\n",
    "     version=1,\n",
    "     description=\"Credit Card Transaction data\",\n",
    "     primary_key=['credit_card_number'],\n",
    "     event_time='trans_datetime'\n",
    ") "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Feature Group created successfully, explore it at \n",
      "https://c.app.hopsworks.ai:443/p/1214673/fs/1202301/fg/1401910\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Uploading Dataframe: 100.00% |██████████| Rows 4/4 | Elapsed Time: 00:00 | Remaining Time: 00:00\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Launching job: credit_card_transactions_1_offline_fg_materialization\n",
      "Job started successfully, you can follow the progress at \n",
      "https://c.app.hopsworks.ai:443/p/1214673/jobs/named/credit_card_transactions_1_offline_fg_materialization/executions\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(Job('credit_card_transactions_1_offline_fg_materialization', 'SPARK'), None)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fg.insert(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Feature view created successfully, explore it at \n",
      "https://c.app.hopsworks.ai:443/p/1214673/fs/1202301/fv/credit_card_transactions/version/1\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<hsfs.feature_view.FeatureView at 0x7fc9e408ca70>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "query = fg.select([\"amount\", \"location\", \"fraud\"])\n",
    "fv = fs.create_feature_view(\n",
    "    name=\"credit_card_transactions\",\n",
    "    version=1,\n",
    "    description=\"Features from credit card transactions\",\n",
    "    labels=[\"fraud\"],\n",
    "    query=query\n",
    ")\n",
    "fv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished: Reading data from Hopsworks, using Hopsworks Feature Query Service (0.32s) \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "VersionWarning: Incremented version to `3`.\n"
     ]
    }
   ],
   "source": [
    "fv = fs.get_feature_view(name='credit_card_transactions', version=1)\n",
    "X_train, X_test, y_train, y_test = fv.train_test_split(0.5)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>amount</th>\n",
       "      <th>location</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>66.29</td>\n",
       "      <td>Stockholm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>142.34</td>\n",
       "      <td>Sao Paolo</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   amount   location\n",
       "1   66.29  Stockholm\n",
       "3  142.34  Sao Paolo"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training dataset job started successfully, you can follow the progress at \n",
      "http://c.app.hopsworks.ai/p/1214673/jobs/named/credit_card_transactions_1_create_fv_td_04032025010015/executions\n",
      "2025-03-04 01:00:25,465 INFO: Waiting for execution to finish. Current state: INITIALIZING. Final status: UNDEFINED\n",
      "2025-03-04 01:00:28,533 INFO: Waiting for execution to finish. Current state: SUBMITTED. Final status: UNDEFINED\n",
      "2025-03-04 01:00:31,600 INFO: Waiting for execution to finish. Current state: RUNNING. Final status: UNDEFINED\n",
      "2025-03-04 01:01:26,791 INFO: Waiting for execution to finish. Current state: AGGREGATING_LOGS. Final status: SUCCEEDED\n",
      "2025-03-04 01:01:26,848 INFO: Waiting for log aggregation to finish.\n",
      "2025-03-04 01:01:35,266 INFO: Execution finished successfully.\n",
      "2025-03-04 01:01:35,267 INFO: Provenance cached data - overwriting last accessed/created training dataset from 3 to 4.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "VersionWarning: Incremented version to `4`.\n"
     ]
    }
   ],
   "source": [
    "td_version, td_job = fv.create_train_test_split(\n",
    "    description = 'Transactions fraud batch training dataset',\n",
    "    data_format = 'csv',\n",
    "    test_size = 0.5,\n",
    "    write_options = {'wait_for_job': True},\n",
    "    coalesce = True,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2025-03-04 01:13:23,759 INFO: Provenance cached data - overwriting last accessed/created training dataset from 1 to 4.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>amount</th>\n",
       "      <th>location</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>66.29</td>\n",
       "      <td>Stockholm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>112.33</td>\n",
       "      <td>Stockholm</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   amount   location\n",
       "0   66.29  Stockholm\n",
       "1  112.33  Stockholm"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train, y_train, X_test, y_test = fv.get_train_test_split(4)\n",
    "X_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Index: 1 entries, 1111 2222 3333 4444 to 1111 2222 3333 4444\n",
      "Data columns (total 1 columns):\n",
      " #   Column       Non-Null Count  Dtype  \n",
      "---  ------       --------------  -----  \n",
      " 0   total_spent  1 non-null      float64\n",
      "dtypes: float64(1)\n",
      "memory usage: 16.0+ bytes\n"
     ]
    }
   ],
   "source": [
    "df2 = df[[\"credit_card_number\", \"amount\"]].groupby(\"credit_card_number\").sum()\n",
    "df2.rename(columns={\"amount\": \"total_spent\"}, inplace=True)\n",
    "df2.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>total_spent</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>credit_card_number</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1111 2222 3333 4444</th>\n",
       "      <td>333.3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     total_spent\n",
       "credit_card_number              \n",
       "1111 2222 3333 4444        333.3"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>total_spent</th>\n",
       "      <th>as_of_datetime</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>credit_card_number</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1111 2222 3333 4444</th>\n",
       "      <td>333.3</td>\n",
       "      <td>2022-01-02 20:55:00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     total_spent      as_of_datetime\n",
       "credit_card_number                                  \n",
       "1111 2222 3333 4444        333.3 2022-01-02 20:55:00"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2[\"as_of_datetime\"] = df[[\"credit_card_number\", \"trans_datetime\"]].groupby(\"credit_card_number\").max()\n",
    "df2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>credit_card_number</th>\n",
       "      <th>total_spent</th>\n",
       "      <th>as_of_datetime</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1111 2222 3333 4444</td>\n",
       "      <td>333.3</td>\n",
       "      <td>2022-01-02 20:55:00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    credit_card_number  total_spent      as_of_datetime\n",
       "0  1111 2222 3333 4444        333.3 2022-01-02 20:55:00"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2.reset_index(inplace=True)\n",
    "df2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<hsfs.feature_group.FeatureGroup at 0x709efe901e20>"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fg2 = fs.get_or_create_feature_group(\n",
    "     name=\"credit_card_spending\",\n",
    "     version=1,\n",
    "     description=\"Credit Card Spending\",\n",
    "     primary_key=['credit_card_number'],\n",
    "     event_time='as_of_datetime'\n",
    ") \n",
    "fg2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Feature Group created successfully, explore it at \n",
      "https://c.app.hopsworks.ai:443/p/1214673/fs/1202301/fg/1403438\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Uploading Dataframe: 100.00% |██████████| Rows 1/1 | Elapsed Time: 00:00 | Remaining Time: 00:00\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Launching job: credit_card_spending_1_offline_fg_materialization\n",
      "Job started successfully, you can follow the progress at \n",
      "https://c.app.hopsworks.ai:443/p/1214673/jobs/named/credit_card_spending_1_offline_fg_materialization/executions\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<hsfs.feature_group.FeatureGroup at 0x709efe901e20>"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fg2.insert(df2, write_options={\"wait_for_job\": False})\n",
    "fg2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Uploading Dataframe: 100.00% |██████████| Rows 4/4 | Elapsed Time: 00:00 | Remaining Time: 00:00\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Launching job: credit_card_transactions_1_offline_fg_materialization\n",
      "Job started successfully, you can follow the progress at \n",
      "https://c.app.hopsworks.ai:443/p/1214673/jobs/named/credit_card_transactions_1_offline_fg_materialization/executions\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(Job('credit_card_transactions_1_offline_fg_materialization', 'SPARK'), None)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "more_data = { \n",
    "    'credit_card_number': ['9999 8888 7777 6666', '9999 8888 7777 6666','9999 8888 7777 6666',\n",
    "                           '9999 8888 7777 6666'],\n",
    "    'trans_datetime': ['2022-01-02 04:11', '2022-01-03 07:24', '2022-01-05 10:33', '2022-01-05 11:50'],\n",
    "    'amount': [55.67, 84, 77.95, 183],\n",
    "    'location': ['San Francisco', 'San Francisco', 'Dublin', 'Dublin'],\n",
    "    'fraud': [False, False, False, False] \n",
    "}\n",
    "\n",
    "df3 = pd.DataFrame.from_dict(more_data)\n",
    "df3['trans_datetime']= pd.to_datetime(df3['trans_datetime'])\n",
    "\n",
    "fg = fs.get_feature_group(name=\"credit_card_transactions\", version=1)\n",
    "\n",
    "fg.insert(df3, write_options={\"wait_for_job\": False})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished: Reading data from Hopsworks, using Hopsworks Feature Query Service (0.56s) \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>credit_card_number</th>\n",
       "      <th>trans_datetime</th>\n",
       "      <th>amount</th>\n",
       "      <th>location</th>\n",
       "      <th>fraud</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1111 2222 3333 4444</td>\n",
       "      <td>2022-01-02 19:44:00+00:00</td>\n",
       "      <td>12.34</td>\n",
       "      <td>Rio De Janeiro</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1111 2222 3333 4444</td>\n",
       "      <td>2022-01-02 20:44:00+00:00</td>\n",
       "      <td>66.29</td>\n",
       "      <td>Stockholm</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1111 2222 3333 4444</td>\n",
       "      <td>2022-01-02 20:55:00+00:00</td>\n",
       "      <td>112.33</td>\n",
       "      <td>Stockholm</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1111 2222 3333 4444</td>\n",
       "      <td>2022-01-01 08:44:00+00:00</td>\n",
       "      <td>142.34</td>\n",
       "      <td>Sao Paolo</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>9999 8888 7777 6666</td>\n",
       "      <td>2022-01-03 07:24:00+00:00</td>\n",
       "      <td>84.00</td>\n",
       "      <td>San Francisco</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>9999 8888 7777 6666</td>\n",
       "      <td>2022-01-05 10:33:00+00:00</td>\n",
       "      <td>77.95</td>\n",
       "      <td>Dublin</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>9999 8888 7777 6666</td>\n",
       "      <td>2022-01-05 11:50:00+00:00</td>\n",
       "      <td>183.00</td>\n",
       "      <td>Dublin</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>9999 8888 7777 6666</td>\n",
       "      <td>2022-01-02 04:11:00+00:00</td>\n",
       "      <td>55.67</td>\n",
       "      <td>San Francisco</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    credit_card_number            trans_datetime  amount        location  \\\n",
       "0  1111 2222 3333 4444 2022-01-02 19:44:00+00:00   12.34  Rio De Janeiro   \n",
       "1  1111 2222 3333 4444 2022-01-02 20:44:00+00:00   66.29       Stockholm   \n",
       "2  1111 2222 3333 4444 2022-01-02 20:55:00+00:00  112.33       Stockholm   \n",
       "3  1111 2222 3333 4444 2022-01-01 08:44:00+00:00  142.34       Sao Paolo   \n",
       "4  9999 8888 7777 6666 2022-01-03 07:24:00+00:00   84.00   San Francisco   \n",
       "5  9999 8888 7777 6666 2022-01-05 10:33:00+00:00   77.95          Dublin   \n",
       "6  9999 8888 7777 6666 2022-01-05 11:50:00+00:00  183.00          Dublin   \n",
       "7  9999 8888 7777 6666 2022-01-02 04:11:00+00:00   55.67   San Francisco   \n",
       "\n",
       "   fraud  \n",
       "0  False  \n",
       "1   True  \n",
       "2   True  \n",
       "3  False  \n",
       "4  False  \n",
       "5  False  \n",
       "6  False  \n",
       "7  False  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fg = fs.get_feature_group(name=\"credit_card_transactions\", version=1)\n",
    "df5 = fg.read()\n",
    "df5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df5 = df5.set_index('trans_datetime')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "df5 = df5.sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>credit_card_number</th>\n",
       "      <th>amount</th>\n",
       "      <th>location</th>\n",
       "      <th>fraud</th>\n",
       "      <th>rolling_max_1d</th>\n",
       "      <th>rolling_min_1d</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>trans_datetime</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2022-01-01 08:44:00+00:00</th>\n",
       "      <td>1111 2222 3333 4444</td>\n",
       "      <td>142.34</td>\n",
       "      <td>Sao Paolo</td>\n",
       "      <td>False</td>\n",
       "      <td>142.34</td>\n",
       "      <td>142.34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2022-01-02 04:11:00+00:00</th>\n",
       "      <td>9999 8888 7777 6666</td>\n",
       "      <td>55.67</td>\n",
       "      <td>San Francisco</td>\n",
       "      <td>False</td>\n",
       "      <td>142.34</td>\n",
       "      <td>55.67</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2022-01-02 19:44:00+00:00</th>\n",
       "      <td>1111 2222 3333 4444</td>\n",
       "      <td>12.34</td>\n",
       "      <td>Rio De Janeiro</td>\n",
       "      <td>False</td>\n",
       "      <td>55.67</td>\n",
       "      <td>12.34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2022-01-02 20:44:00+00:00</th>\n",
       "      <td>1111 2222 3333 4444</td>\n",
       "      <td>66.29</td>\n",
       "      <td>Stockholm</td>\n",
       "      <td>True</td>\n",
       "      <td>66.29</td>\n",
       "      <td>12.34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2022-01-02 20:55:00+00:00</th>\n",
       "      <td>1111 2222 3333 4444</td>\n",
       "      <td>112.33</td>\n",
       "      <td>Stockholm</td>\n",
       "      <td>True</td>\n",
       "      <td>112.33</td>\n",
       "      <td>12.34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2022-01-03 07:24:00+00:00</th>\n",
       "      <td>9999 8888 7777 6666</td>\n",
       "      <td>84.00</td>\n",
       "      <td>San Francisco</td>\n",
       "      <td>False</td>\n",
       "      <td>112.33</td>\n",
       "      <td>12.34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2022-01-05 10:33:00+00:00</th>\n",
       "      <td>9999 8888 7777 6666</td>\n",
       "      <td>77.95</td>\n",
       "      <td>Dublin</td>\n",
       "      <td>False</td>\n",
       "      <td>77.95</td>\n",
       "      <td>77.95</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2022-01-05 11:50:00+00:00</th>\n",
       "      <td>9999 8888 7777 6666</td>\n",
       "      <td>183.00</td>\n",
       "      <td>Dublin</td>\n",
       "      <td>False</td>\n",
       "      <td>183.00</td>\n",
       "      <td>77.95</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                            credit_card_number  amount        location  fraud  \\\n",
       "trans_datetime                                                                  \n",
       "2022-01-01 08:44:00+00:00  1111 2222 3333 4444  142.34       Sao Paolo  False   \n",
       "2022-01-02 04:11:00+00:00  9999 8888 7777 6666   55.67   San Francisco  False   \n",
       "2022-01-02 19:44:00+00:00  1111 2222 3333 4444   12.34  Rio De Janeiro  False   \n",
       "2022-01-02 20:44:00+00:00  1111 2222 3333 4444   66.29       Stockholm   True   \n",
       "2022-01-02 20:55:00+00:00  1111 2222 3333 4444  112.33       Stockholm   True   \n",
       "2022-01-03 07:24:00+00:00  9999 8888 7777 6666   84.00   San Francisco  False   \n",
       "2022-01-05 10:33:00+00:00  9999 8888 7777 6666   77.95          Dublin  False   \n",
       "2022-01-05 11:50:00+00:00  9999 8888 7777 6666  183.00          Dublin  False   \n",
       "\n",
       "                           rolling_max_1d  rolling_min_1d  \n",
       "trans_datetime                                             \n",
       "2022-01-01 08:44:00+00:00          142.34          142.34  \n",
       "2022-01-02 04:11:00+00:00          142.34           55.67  \n",
       "2022-01-02 19:44:00+00:00           55.67           12.34  \n",
       "2022-01-02 20:44:00+00:00           66.29           12.34  \n",
       "2022-01-02 20:55:00+00:00          112.33           12.34  \n",
       "2022-01-03 07:24:00+00:00          112.33           12.34  \n",
       "2022-01-05 10:33:00+00:00           77.95           77.95  \n",
       "2022-01-05 11:50:00+00:00          183.00           77.95  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df5['rolling_max_1d'] = df5.rolling('1D').amount.max()\n",
    "df5['rolling_min_1d'] = df5.rolling('1D').amount.min()\n",
    "df5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "df5.reset_index(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "fg_agg = fs.get_or_create_feature_group(\n",
    "     name=\"credit_card_rolling_windows\",\n",
    "     version=1,\n",
    "     description=\"Daily Credit Card Spending\",\n",
    "     primary_key=['credit_card_number'],\n",
    "     event_time='trans_datetime'\n",
    ") "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Feature Group created successfully, explore it at \n",
      "https://c.app.hopsworks.ai:443/p/1214673/fs/1202301/fg/1403521\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Uploading Dataframe: 100.00% |██████████| Rows 8/8 | Elapsed Time: 00:00 | Remaining Time: 00:00\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Launching job: credit_card_rolling_windows_1_offline_fg_materialization\n",
      "Job started successfully, you can follow the progress at \n",
      "https://c.app.hopsworks.ai:443/p/1214673/jobs/named/credit_card_rolling_windows_1_offline_fg_materialization/executions\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(Job('credit_card_rolling_windows_1_offline_fg_materialization', 'SPARK'),\n",
       " None)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fg_agg.insert(df5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2025-03-05 03:02:23,834 ERROR: Peer ServerlessFstore__majidali is not known. Please register client certificates first.. Detail: Python exception: FlyingDuckException. gRPC client debug context: UNKNOWN:Error received from peer ipv4:51.79.26.27:5005 {created_time:\"2025-03-05T03:02:23.834713087+00:00\", grpc_status:2, grpc_message:\"Peer ServerlessFstore__majidali is not known. Please register client certificates first.. Detail: Python exception: FlyingDuckException\"}. Client context: IOError: Server never sent a data message. Detail: Internal\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/codespace/.pyenv/versions/3.9.21/lib/python3.9/site-packages/hsfs/core/arrow_flight_client.py\", line 394, in afs_error_handler_wrapper\n",
      "    return func(instance, *args, **kw)\n",
      "  File \"/home/codespace/.pyenv/versions/3.9.21/lib/python3.9/site-packages/hsfs/core/arrow_flight_client.py\", line 459, in read_query\n",
      "    return self._get_dataset(\n",
      "  File \"/home/codespace/.pyenv/versions/3.9.21/lib/python3.9/site-packages/retrying.py\", line 56, in wrapped_f\n",
      "    return Retrying(*dargs, **dkw).call(f, *args, **kw)\n",
      "  File \"/home/codespace/.pyenv/versions/3.9.21/lib/python3.9/site-packages/retrying.py\", line 257, in call\n",
      "    return attempt.get(self._wrap_exception)\n",
      "  File \"/home/codespace/.pyenv/versions/3.9.21/lib/python3.9/site-packages/retrying.py\", line 301, in get\n",
      "    six.reraise(self.value[0], self.value[1], self.value[2])\n",
      "  File \"/home/codespace/.pyenv/versions/3.9.21/lib/python3.9/site-packages/six.py\", line 724, in reraise\n",
      "    raise value\n",
      "  File \"/home/codespace/.pyenv/versions/3.9.21/lib/python3.9/site-packages/retrying.py\", line 251, in call\n",
      "    attempt = Attempt(fn(*args, **kwargs), attempt_number, False)\n",
      "  File \"/home/codespace/.pyenv/versions/3.9.21/lib/python3.9/site-packages/hsfs/core/arrow_flight_client.py\", line 445, in _get_dataset\n",
      "    reader = self._connection.do_get(info.endpoints[0].ticket, options)\n",
      "  File \"pyarrow/_flight.pyx\", line 1708, in pyarrow._flight.FlightClient.do_get\n",
      "  File \"pyarrow/_flight.pyx\", line 58, in pyarrow._flight.check_flight_status\n",
      "pyarrow._flight.FlightServerError: Peer ServerlessFstore__majidali is not known. Please register client certificates first.. Detail: Python exception: FlyingDuckException. gRPC client debug context: UNKNOWN:Error received from peer ipv4:51.79.26.27:5005 {created_time:\"2025-03-05T03:02:23.834713087+00:00\", grpc_status:2, grpc_message:\"Peer ServerlessFstore__majidali is not known. Please register client certificates first.. Detail: Python exception: FlyingDuckException\"}. Client context: IOError: Server never sent a data message. Detail: Internal\n",
      "Finished: Reading data from Hopsworks, using Hopsworks Feature Query Service (1.14s) \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>credit_card_number</th>\n",
       "      <th>trans_datetime</th>\n",
       "      <th>amount</th>\n",
       "      <th>location</th>\n",
       "      <th>fraud</th>\n",
       "      <th>rolling_max_1d</th>\n",
       "      <th>rolling_min_1d</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9999 8888 7777 6666</td>\n",
       "      <td>2022-01-02 04:11:00+00:00</td>\n",
       "      <td>55.67</td>\n",
       "      <td>San Francisco</td>\n",
       "      <td>False</td>\n",
       "      <td>142.34</td>\n",
       "      <td>55.67</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9999 8888 7777 6666</td>\n",
       "      <td>2022-01-03 07:24:00+00:00</td>\n",
       "      <td>84.00</td>\n",
       "      <td>San Francisco</td>\n",
       "      <td>False</td>\n",
       "      <td>112.33</td>\n",
       "      <td>12.34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9999 8888 7777 6666</td>\n",
       "      <td>2022-01-05 10:33:00+00:00</td>\n",
       "      <td>77.95</td>\n",
       "      <td>Dublin</td>\n",
       "      <td>False</td>\n",
       "      <td>77.95</td>\n",
       "      <td>77.95</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9999 8888 7777 6666</td>\n",
       "      <td>2022-01-05 11:50:00+00:00</td>\n",
       "      <td>183.00</td>\n",
       "      <td>Dublin</td>\n",
       "      <td>False</td>\n",
       "      <td>183.00</td>\n",
       "      <td>77.95</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1111 2222 3333 4444</td>\n",
       "      <td>2022-01-01 08:44:00+00:00</td>\n",
       "      <td>142.34</td>\n",
       "      <td>Sao Paolo</td>\n",
       "      <td>False</td>\n",
       "      <td>142.34</td>\n",
       "      <td>142.34</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    credit_card_number            trans_datetime  amount       location  \\\n",
       "0  9999 8888 7777 6666 2022-01-02 04:11:00+00:00   55.67  San Francisco   \n",
       "1  9999 8888 7777 6666 2022-01-03 07:24:00+00:00   84.00  San Francisco   \n",
       "2  9999 8888 7777 6666 2022-01-05 10:33:00+00:00   77.95         Dublin   \n",
       "3  9999 8888 7777 6666 2022-01-05 11:50:00+00:00  183.00         Dublin   \n",
       "4  1111 2222 3333 4444 2022-01-01 08:44:00+00:00  142.34      Sao Paolo   \n",
       "\n",
       "   fraud  rolling_max_1d  rolling_min_1d  \n",
       "0  False          142.34           55.67  \n",
       "1  False          112.33           12.34  \n",
       "2  False           77.95           77.95  \n",
       "3  False          183.00           77.95  \n",
       "4  False          142.34          142.34  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "query = fg.select_all().join(fg_agg.select(['rolling_max_1d', 'rolling_min_1d']))\n",
    "\n",
    "training_data = query.read()\n",
    "training_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.21"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}