2024-heraklion-data/notebooks/030_tabular_data/041_window_functions_tutor.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "247bbf84",
   "metadata": {},
   "source": [
    "# Window functions for tabular data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "44584190",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "83bbd275",
   "metadata": {},
   "source": [
    "# Load experimental data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "88b9e189",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('timed_responses.csv', index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "987a3518",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>subject_id</th>\n",
       "      <th>time (ms)</th>\n",
       "      <th>response</th>\n",
       "      <th>accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>574</th>\n",
       "      <td>3</td>\n",
       "      <td>540</td>\n",
       "      <td>RIGHT</td>\n",
       "      <td>0.04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1190</th>\n",
       "      <td>2</td>\n",
       "      <td>552</td>\n",
       "      <td>LEFT</td>\n",
       "      <td>0.43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1895</th>\n",
       "      <td>2</td>\n",
       "      <td>1036</td>\n",
       "      <td>LEFT</td>\n",
       "      <td>0.36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>3</td>\n",
       "      <td>257</td>\n",
       "      <td>RIGHT</td>\n",
       "      <td>0.11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>158</th>\n",
       "      <td>2</td>\n",
       "      <td>743</td>\n",
       "      <td>RIGHT</td>\n",
       "      <td>0.32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>551</th>\n",
       "      <td>3</td>\n",
       "      <td>619</td>\n",
       "      <td>LEFT</td>\n",
       "      <td>0.25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1602</th>\n",
       "      <td>1</td>\n",
       "      <td>43</td>\n",
       "      <td>RIGHT</td>\n",
       "      <td>0.65</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>413</th>\n",
       "      <td>1</td>\n",
       "      <td>471</td>\n",
       "      <td>LEFT</td>\n",
       "      <td>0.80</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>785</th>\n",
       "      <td>1</td>\n",
       "      <td>121</td>\n",
       "      <td>LEFT</td>\n",
       "      <td>0.10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1393</th>\n",
       "      <td>2</td>\n",
       "      <td>903</td>\n",
       "      <td>RIGHT</td>\n",
       "      <td>0.33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>629</th>\n",
       "      <td>2</td>\n",
       "      <td>353</td>\n",
       "      <td>LEFT</td>\n",
       "      <td>0.17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1829</th>\n",
       "      <td>3</td>\n",
       "      <td>768</td>\n",
       "      <td>RIGHT</td>\n",
       "      <td>0.26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>902</th>\n",
       "      <td>1</td>\n",
       "      <td>1093</td>\n",
       "      <td>LEFT</td>\n",
       "      <td>0.34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1486</th>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>RIGHT</td>\n",
       "      <td>0.29</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      subject_id  time (ms) response  accuracy\n",
       "574            3        540    RIGHT      0.04\n",
       "1190           2        552     LEFT      0.43\n",
       "1895           2       1036     LEFT      0.36\n",
       "53             3        257    RIGHT      0.11\n",
       "158            2        743    RIGHT      0.32\n",
       "551            3        619     LEFT      0.25\n",
       "1602           1         43    RIGHT      0.65\n",
       "413            1        471     LEFT      0.80\n",
       "785            1        121     LEFT      0.10\n",
       "1393           2        903    RIGHT      0.33\n",
       "629            2        353     LEFT      0.17\n",
       "1829           3        768    RIGHT      0.26\n",
       "902            1       1093     LEFT      0.34\n",
       "1486           2          3    RIGHT      0.29"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5c41cd93",
   "metadata": {},
   "source": [
    "# Split-apply-combine operations return one aggregated value per group"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0234ccf2",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.groupby('subject_id')['accuracy'].max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2b2a1796",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "2bb99152",
   "metadata": {},
   "source": [
    "# However, for some calculations we need to have a value per row\n",
    "\n",
    "For example: for each subject, rank the responses by decreasing accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3aed0755",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "17f3d40f",
   "metadata": {},
   "source": [
    "# In many cases, a window functions is combined with a sorting operation\n",
    "\n",
    "For example: for each subject, count the number of \"LEFT\" responses up until any moment in the experiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "67efdd56",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "a00b4f39",
   "metadata": {},
   "source": [
    "# Window functions are also useful to compute changes in the data for each group\n",
    "\n",
    "In this case, the window function often uses the `shift(n)` method that lags the data by `n` rows"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e553c17f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f2973e3d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c9ca46b0",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}