321 lines
7.8 KiB
Plaintext
321 lines
7.8 KiB
Plaintext
|
{
|
||
|
"cells": [
|
||
|
{
|
||
|
"cell_type": "markdown",
|
||
|
"id": "247bbf84",
|
||
|
"metadata": {},
|
||
|
"source": [
|
||
|
"# Window functions for tabular data"
|
||
|
]
|
||
|
},
|
||
|
{
|
||
|
"cell_type": "code",
|
||
|
"execution_count": 1,
|
||
|
"id": "44584190",
|
||
|
"metadata": {},
|
||
|
"outputs": [],
|
||
|
"source": [
|
||
|
"import pandas as pd"
|
||
|
]
|
||
|
},
|
||
|
{
|
||
|
"cell_type": "markdown",
|
||
|
"id": "83bbd275",
|
||
|
"metadata": {},
|
||
|
"source": [
|
||
|
"# Load experimental data"
|
||
|
]
|
||
|
},
|
||
|
{
|
||
|
"cell_type": "code",
|
||
|
"execution_count": 2,
|
||
|
"id": "88b9e189",
|
||
|
"metadata": {},
|
||
|
"outputs": [],
|
||
|
"source": [
|
||
|
"df = pd.read_csv('timed_responses.csv', index_col=0)"
|
||
|
]
|
||
|
},
|
||
|
{
|
||
|
"cell_type": "code",
|
||
|
"execution_count": 3,
|
||
|
"id": "987a3518",
|
||
|
"metadata": {},
|
||
|
"outputs": [
|
||
|
{
|
||
|
"data": {
|
||
|
"text/html": [
|
||
|
"<div>\n",
|
||
|
"<style scoped>\n",
|
||
|
" .dataframe tbody tr th:only-of-type {\n",
|
||
|
" vertical-align: middle;\n",
|
||
|
" }\n",
|
||
|
"\n",
|
||
|
" .dataframe tbody tr th {\n",
|
||
|
" vertical-align: top;\n",
|
||
|
" }\n",
|
||
|
"\n",
|
||
|
" .dataframe thead th {\n",
|
||
|
" text-align: right;\n",
|
||
|
" }\n",
|
||
|
"</style>\n",
|
||
|
"<table border=\"1\" class=\"dataframe\">\n",
|
||
|
" <thead>\n",
|
||
|
" <tr style=\"text-align: right;\">\n",
|
||
|
" <th></th>\n",
|
||
|
" <th>subject_id</th>\n",
|
||
|
" <th>time (ms)</th>\n",
|
||
|
" <th>response</th>\n",
|
||
|
" <th>accuracy</th>\n",
|
||
|
" </tr>\n",
|
||
|
" </thead>\n",
|
||
|
" <tbody>\n",
|
||
|
" <tr>\n",
|
||
|
" <th>574</th>\n",
|
||
|
" <td>3</td>\n",
|
||
|
" <td>540</td>\n",
|
||
|
" <td>RIGHT</td>\n",
|
||
|
" <td>0.04</td>\n",
|
||
|
" </tr>\n",
|
||
|
" <tr>\n",
|
||
|
" <th>1190</th>\n",
|
||
|
" <td>2</td>\n",
|
||
|
" <td>552</td>\n",
|
||
|
" <td>LEFT</td>\n",
|
||
|
" <td>0.43</td>\n",
|
||
|
" </tr>\n",
|
||
|
" <tr>\n",
|
||
|
" <th>1895</th>\n",
|
||
|
" <td>2</td>\n",
|
||
|
" <td>1036</td>\n",
|
||
|
" <td>LEFT</td>\n",
|
||
|
" <td>0.36</td>\n",
|
||
|
" </tr>\n",
|
||
|
" <tr>\n",
|
||
|
" <th>53</th>\n",
|
||
|
" <td>3</td>\n",
|
||
|
" <td>257</td>\n",
|
||
|
" <td>RIGHT</td>\n",
|
||
|
" <td>0.11</td>\n",
|
||
|
" </tr>\n",
|
||
|
" <tr>\n",
|
||
|
" <th>158</th>\n",
|
||
|
" <td>2</td>\n",
|
||
|
" <td>743</td>\n",
|
||
|
" <td>RIGHT</td>\n",
|
||
|
" <td>0.32</td>\n",
|
||
|
" </tr>\n",
|
||
|
" <tr>\n",
|
||
|
" <th>551</th>\n",
|
||
|
" <td>3</td>\n",
|
||
|
" <td>619</td>\n",
|
||
|
" <td>LEFT</td>\n",
|
||
|
" <td>0.25</td>\n",
|
||
|
" </tr>\n",
|
||
|
" <tr>\n",
|
||
|
" <th>1602</th>\n",
|
||
|
" <td>1</td>\n",
|
||
|
" <td>43</td>\n",
|
||
|
" <td>RIGHT</td>\n",
|
||
|
" <td>0.65</td>\n",
|
||
|
" </tr>\n",
|
||
|
" <tr>\n",
|
||
|
" <th>413</th>\n",
|
||
|
" <td>1</td>\n",
|
||
|
" <td>471</td>\n",
|
||
|
" <td>LEFT</td>\n",
|
||
|
" <td>0.80</td>\n",
|
||
|
" </tr>\n",
|
||
|
" <tr>\n",
|
||
|
" <th>785</th>\n",
|
||
|
" <td>1</td>\n",
|
||
|
" <td>121</td>\n",
|
||
|
" <td>LEFT</td>\n",
|
||
|
" <td>0.10</td>\n",
|
||
|
" </tr>\n",
|
||
|
" <tr>\n",
|
||
|
" <th>1393</th>\n",
|
||
|
" <td>2</td>\n",
|
||
|
" <td>903</td>\n",
|
||
|
" <td>RIGHT</td>\n",
|
||
|
" <td>0.33</td>\n",
|
||
|
" </tr>\n",
|
||
|
" <tr>\n",
|
||
|
" <th>629</th>\n",
|
||
|
" <td>2</td>\n",
|
||
|
" <td>353</td>\n",
|
||
|
" <td>LEFT</td>\n",
|
||
|
" <td>0.17</td>\n",
|
||
|
" </tr>\n",
|
||
|
" <tr>\n",
|
||
|
" <th>1829</th>\n",
|
||
|
" <td>3</td>\n",
|
||
|
" <td>768</td>\n",
|
||
|
" <td>RIGHT</td>\n",
|
||
|
" <td>0.26</td>\n",
|
||
|
" </tr>\n",
|
||
|
" <tr>\n",
|
||
|
" <th>902</th>\n",
|
||
|
" <td>1</td>\n",
|
||
|
" <td>1093</td>\n",
|
||
|
" <td>LEFT</td>\n",
|
||
|
" <td>0.34</td>\n",
|
||
|
" </tr>\n",
|
||
|
" <tr>\n",
|
||
|
" <th>1486</th>\n",
|
||
|
" <td>2</td>\n",
|
||
|
" <td>3</td>\n",
|
||
|
" <td>RIGHT</td>\n",
|
||
|
" <td>0.29</td>\n",
|
||
|
" </tr>\n",
|
||
|
" </tbody>\n",
|
||
|
"</table>\n",
|
||
|
"</div>"
|
||
|
],
|
||
|
"text/plain": [
|
||
|
" subject_id time (ms) response accuracy\n",
|
||
|
"574 3 540 RIGHT 0.04\n",
|
||
|
"1190 2 552 LEFT 0.43\n",
|
||
|
"1895 2 1036 LEFT 0.36\n",
|
||
|
"53 3 257 RIGHT 0.11\n",
|
||
|
"158 2 743 RIGHT 0.32\n",
|
||
|
"551 3 619 LEFT 0.25\n",
|
||
|
"1602 1 43 RIGHT 0.65\n",
|
||
|
"413 1 471 LEFT 0.80\n",
|
||
|
"785 1 121 LEFT 0.10\n",
|
||
|
"1393 2 903 RIGHT 0.33\n",
|
||
|
"629 2 353 LEFT 0.17\n",
|
||
|
"1829 3 768 RIGHT 0.26\n",
|
||
|
"902 1 1093 LEFT 0.34\n",
|
||
|
"1486 2 3 RIGHT 0.29"
|
||
|
]
|
||
|
},
|
||
|
"execution_count": 3,
|
||
|
"metadata": {},
|
||
|
"output_type": "execute_result"
|
||
|
}
|
||
|
],
|
||
|
"source": [
|
||
|
"df"
|
||
|
]
|
||
|
},
|
||
|
{
|
||
|
"cell_type": "markdown",
|
||
|
"id": "5c41cd93",
|
||
|
"metadata": {},
|
||
|
"source": [
|
||
|
"# Split-apply-combine operations return one aggregated value per group"
|
||
|
]
|
||
|
},
|
||
|
{
|
||
|
"cell_type": "code",
|
||
|
"execution_count": null,
|
||
|
"id": "0234ccf2",
|
||
|
"metadata": {},
|
||
|
"outputs": [],
|
||
|
"source": [
|
||
|
"df.groupby('subject_id')['accuracy'].max()"
|
||
|
]
|
||
|
},
|
||
|
{
|
||
|
"cell_type": "code",
|
||
|
"execution_count": null,
|
||
|
"id": "2b2a1796",
|
||
|
"metadata": {},
|
||
|
"outputs": [],
|
||
|
"source": []
|
||
|
},
|
||
|
{
|
||
|
"cell_type": "markdown",
|
||
|
"id": "2bb99152",
|
||
|
"metadata": {},
|
||
|
"source": [
|
||
|
"# However, for some calculations we need to have a value per row\n",
|
||
|
"\n",
|
||
|
"For example: for each subject, rank the responses by decreasing accuracy"
|
||
|
]
|
||
|
},
|
||
|
{
|
||
|
"cell_type": "code",
|
||
|
"execution_count": null,
|
||
|
"id": "3aed0755",
|
||
|
"metadata": {},
|
||
|
"outputs": [],
|
||
|
"source": []
|
||
|
},
|
||
|
{
|
||
|
"cell_type": "markdown",
|
||
|
"id": "17f3d40f",
|
||
|
"metadata": {},
|
||
|
"source": [
|
||
|
"# In many cases, a window functions is combined with a sorting operation\n",
|
||
|
"\n",
|
||
|
"For example: for each subject, count the number of \"LEFT\" responses up until any moment in the experiment"
|
||
|
]
|
||
|
},
|
||
|
{
|
||
|
"cell_type": "code",
|
||
|
"execution_count": null,
|
||
|
"id": "67efdd56",
|
||
|
"metadata": {},
|
||
|
"outputs": [],
|
||
|
"source": []
|
||
|
},
|
||
|
{
|
||
|
"cell_type": "markdown",
|
||
|
"id": "a00b4f39",
|
||
|
"metadata": {},
|
||
|
"source": [
|
||
|
"# Window functions are also useful to compute changes in the data for each group\n",
|
||
|
"\n",
|
||
|
"In this case, the window function often uses the `shift(n)` method that lags the data by `n` rows"
|
||
|
]
|
||
|
},
|
||
|
{
|
||
|
"cell_type": "code",
|
||
|
"execution_count": null,
|
||
|
"id": "e553c17f",
|
||
|
"metadata": {},
|
||
|
"outputs": [],
|
||
|
"source": []
|
||
|
},
|
||
|
{
|
||
|
"cell_type": "code",
|
||
|
"execution_count": null,
|
||
|
"id": "f2973e3d",
|
||
|
"metadata": {},
|
||
|
"outputs": [],
|
||
|
"source": []
|
||
|
},
|
||
|
{
|
||
|
"cell_type": "code",
|
||
|
"execution_count": null,
|
||
|
"id": "c9ca46b0",
|
||
|
"metadata": {},
|
||
|
"outputs": [],
|
||
|
"source": []
|
||
|
}
|
||
|
],
|
||
|
"metadata": {
|
||
|
"kernelspec": {
|
||
|
"display_name": "Python 3 (ipykernel)",
|
||
|
"language": "python",
|
||
|
"name": "python3"
|
||
|
},
|
||
|
"language_info": {
|
||
|
"codemirror_mode": {
|
||
|
"name": "ipython",
|
||
|
"version": 3
|
||
|
},
|
||
|
"file_extension": ".py",
|
||
|
"mimetype": "text/x-python",
|
||
|
"name": "python",
|
||
|
"nbconvert_exporter": "python",
|
||
|
"pygments_lexer": "ipython3",
|
||
|
"version": "3.11.3"
|
||
|
}
|
||
|
},
|
||
|
"nbformat": 4,
|
||
|
"nbformat_minor": 5
|
||
|
}
|