ASPP/2024-heraklion-data
8
1
Fork 20
Code Issues 7 Pull requests 2 Actions Packages Projects Releases Wiki Activity
2024-heraklion-data/notebooks/030_tabular_data/040_window_functions.ipynb

1430 lines
41 KiB
Plaintext
Raw Normal View History

ASPP 2024 material
2024-08-27 14:27:53 +02:00
{
"cells": [
{
"cell_type": "markdown",
"id": "247bbf84",
"metadata": {},
"source": [
"# Window functions for tabular data"
]
},
{
"cell_type": "code",
"execution_count": 1,
"id": "44584190",
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd"
]
},
{
"cell_type": "markdown",
"id": "8c3508da",
"metadata": {},
"source": [
"# Load experimental data"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "8e22f6d4",
"metadata": {},
"outputs": [],
"source": [
"df = pd.read_csv('timed_responses.csv', index_col=0)"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "c4504d72",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>subject_id</th>\n",
" <th>time (ms)</th>\n",
" <th>response</th>\n",
" <th>accuracy</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>574</th>\n",
" <td>3</td>\n",
" <td>540</td>\n",
" <td>RIGHT</td>\n",
" <td>0.04</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1190</th>\n",
" <td>2</td>\n",
" <td>552</td>\n",
" <td>LEFT</td>\n",
" <td>0.43</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1895</th>\n",
" <td>2</td>\n",
" <td>1036</td>\n",
" <td>LEFT</td>\n",
" <td>0.36</td>\n",
" </tr>\n",
" <tr>\n",
" <th>53</th>\n",
" <td>3</td>\n",
" <td>257</td>\n",
" <td>RIGHT</td>\n",
" <td>0.11</td>\n",
" </tr>\n",
" <tr>\n",
" <th>158</th>\n",
" <td>2</td>\n",
" <td>743</td>\n",
" <td>RIGHT</td>\n",
" <td>0.32</td>\n",
" </tr>\n",
" <tr>\n",
" <th>551</th>\n",
" <td>3</td>\n",
" <td>619</td>\n",
" <td>LEFT</td>\n",
" <td>0.25</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1602</th>\n",
" <td>1</td>\n",
" <td>43</td>\n",
" <td>RIGHT</td>\n",
" <td>0.65</td>\n",
" </tr>\n",
" <tr>\n",
" <th>413</th>\n",
" <td>1</td>\n",
" <td>471</td>\n",
" <td>LEFT</td>\n",
" <td>0.80</td>\n",
" </tr>\n",
" <tr>\n",
" <th>785</th>\n",
" <td>1</td>\n",
" <td>121</td>\n",
" <td>LEFT</td>\n",
" <td>0.10</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1393</th>\n",
" <td>2</td>\n",
" <td>903</td>\n",
" <td>RIGHT</td>\n",
" <td>0.33</td>\n",
" </tr>\n",
" <tr>\n",
" <th>629</th>\n",
" <td>2</td>\n",
" <td>353</td>\n",
" <td>LEFT</td>\n",
" <td>0.17</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1829</th>\n",
" <td>3</td>\n",
" <td>768</td>\n",
" <td>RIGHT</td>\n",
" <td>0.26</td>\n",
" </tr>\n",
" <tr>\n",
" <th>902</th>\n",
" <td>1</td>\n",
" <td>1093</td>\n",
" <td>LEFT</td>\n",
" <td>0.34</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1486</th>\n",
" <td>2</td>\n",
" <td>3</td>\n",
" <td>RIGHT</td>\n",
" <td>0.29</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" subject_id time (ms) response accuracy\n",
"574 3 540 RIGHT 0.04\n",
"1190 2 552 LEFT 0.43\n",
"1895 2 1036 LEFT 0.36\n",
"53 3 257 RIGHT 0.11\n",
"158 2 743 RIGHT 0.32\n",
"551 3 619 LEFT 0.25\n",
"1602 1 43 RIGHT 0.65\n",
"413 1 471 LEFT 0.80\n",
"785 1 121 LEFT 0.10\n",
"1393 2 903 RIGHT 0.33\n",
"629 2 353 LEFT 0.17\n",
"1829 3 768 RIGHT 0.26\n",
"902 1 1093 LEFT 0.34\n",
"1486 2 3 RIGHT 0.29"
]
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"df"
]
},
{
"cell_type": "markdown",
"id": "a72f45c6",
"metadata": {},
"source": [
"# Split-apply-combine operations return one aggregated value per group"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "0234ccf2",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"subject_id\n",
"1 0.80\n",
"2 0.43\n",
"3 0.26\n",
"Name: accuracy, dtype: float64"
]
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"df.groupby('subject_id')['accuracy'].max()"
]
},
{
"cell_type": "markdown",
"id": "b8926b52",
"metadata": {},
"source": [
"# However, for some calculations we need to have a value per row\n",
"\n",
"For example: for each subject, rank the responses by decreasing accuracy"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "0c77c2dc",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"574 1.0\n",
"1190 6.0\n",
"1895 5.0\n",
"53 2.0\n",
"158 3.0\n",
"551 3.0\n",
"1602 3.0\n",
"413 4.0\n",
"785 1.0\n",
"1393 4.0\n",
"629 1.0\n",
"1829 4.0\n",
"902 2.0\n",
"1486 2.0\n",
"Name: accuracy, dtype: float64"
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"df.groupby('subject_id')['accuracy'].rank()"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "6803bea3",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>subject_id</th>\n",
" <th>time (ms)</th>\n",
" <th>response</th>\n",
" <th>accuracy</th>\n",
" <th>accuracy_rank</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>574</th>\n",
" <td>3</td>\n",
" <td>540</td>\n",
" <td>RIGHT</td>\n",
" <td>0.04</td>\n",
" <td>4.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1190</th>\n",
" <td>2</td>\n",
" <td>552</td>\n",
" <td>LEFT</td>\n",
" <td>0.43</td>\n",
" <td>1.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1895</th>\n",
" <td>2</td>\n",
" <td>1036</td>\n",
" <td>LEFT</td>\n",
" <td>0.36</td>\n",
" <td>2.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>53</th>\n",
" <td>3</td>\n",
" <td>257</td>\n",
" <td>RIGHT</td>\n",
" <td>0.11</td>\n",
" <td>3.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>158</th>\n",
" <td>2</td>\n",
" <td>743</td>\n",
" <td>RIGHT</td>\n",
" <td>0.32</td>\n",
" <td>4.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>551</th>\n",
" <td>3</td>\n",
" <td>619</td>\n",
" <td>LEFT</td>\n",
" <td>0.25</td>\n",
" <td>2.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1602</th>\n",
" <td>1</td>\n",
" <td>43</td>\n",
" <td>RIGHT</td>\n",
" <td>0.65</td>\n",
" <td>2.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>413</th>\n",
" <td>1</td>\n",
" <td>471</td>\n",
" <td>LEFT</td>\n",
" <td>0.80</td>\n",
" <td>1.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>785</th>\n",
" <td>1</td>\n",
" <td>121</td>\n",
" <td>LEFT</td>\n",
" <td>0.10</td>\n",
" <td>4.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1393</th>\n",
" <td>2</td>\n",
" <td>903</td>\n",
" <td>RIGHT</td>\n",
" <td>0.33</td>\n",
" <td>3.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>629</th>\n",
" <td>2</td>\n",
" <td>353</td>\n",
" <td>LEFT</td>\n",
" <td>0.17</td>\n",
" <td>6.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1829</th>\n",
" <td>3</td>\n",
" <td>768</td>\n",
" <td>RIGHT</td>\n",
" <td>0.26</td>\n",
" <td>1.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>902</th>\n",
" <td>1</td>\n",
" <td>1093</td>\n",
" <td>LEFT</td>\n",
" <td>0.34</td>\n",
" <td>3.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1486</th>\n",
" <td>2</td>\n",
" <td>3</td>\n",
" <td>RIGHT</td>\n",
" <td>0.29</td>\n",
" <td>5.0</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" subject_id time (ms) response accuracy accuracy_rank\n",
"574 3 540 RIGHT 0.04 4.0\n",
"1190 2 552 LEFT 0.43 1.0\n",
"1895 2 1036 LEFT 0.36 2.0\n",
"53 3 257 RIGHT 0.11 3.0\n",
"158 2 743 RIGHT 0.32 4.0\n",
"551 3 619 LEFT 0.25 2.0\n",
"1602 1 43 RIGHT 0.65 2.0\n",
"413 1 471 LEFT 0.80 1.0\n",
"785 1 121 LEFT 0.10 4.0\n",
"1393 2 903 RIGHT 0.33 3.0\n",
"629 2 353 LEFT 0.17 6.0\n",
"1829 3 768 RIGHT 0.26 1.0\n",
"902 1 1093 LEFT 0.34 3.0\n",
"1486 2 3 RIGHT 0.29 5.0"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"df['accuracy_rank'] = df.groupby('subject_id')['accuracy'].rank(ascending=False)\n",
"df"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "5690feee",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>subject_id</th>\n",
" <th>time (ms)</th>\n",
" <th>response</th>\n",
" <th>accuracy</th>\n",
" <th>accuracy_rank</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>413</th>\n",
" <td>1</td>\n",
" <td>471</td>\n",
" <td>LEFT</td>\n",
" <td>0.80</td>\n",
" <td>1.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1602</th>\n",
" <td>1</td>\n",
" <td>43</td>\n",
" <td>RIGHT</td>\n",
" <td>0.65</td>\n",
" <td>2.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>902</th>\n",
" <td>1</td>\n",
" <td>1093</td>\n",
" <td>LEFT</td>\n",
" <td>0.34</td>\n",
" <td>3.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>785</th>\n",
" <td>1</td>\n",
" <td>121</td>\n",
" <td>LEFT</td>\n",
" <td>0.10</td>\n",
" <td>4.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1190</th>\n",
" <td>2</td>\n",
" <td>552</td>\n",
" <td>LEFT</td>\n",
" <td>0.43</td>\n",
" <td>1.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1895</th>\n",
" <td>2</td>\n",
" <td>1036</td>\n",
" <td>LEFT</td>\n",
" <td>0.36</td>\n",
" <td>2.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1393</th>\n",
" <td>2</td>\n",
" <td>903</td>\n",
" <td>RIGHT</td>\n",
" <td>0.33</td>\n",
" <td>3.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>158</th>\n",
" <td>2</td>\n",
" <td>743</td>\n",
" <td>RIGHT</td>\n",
" <td>0.32</td>\n",
" <td>4.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1486</th>\n",
" <td>2</td>\n",
" <td>3</td>\n",
" <td>RIGHT</td>\n",
" <td>0.29</td>\n",
" <td>5.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>629</th>\n",
" <td>2</td>\n",
" <td>353</td>\n",
" <td>LEFT</td>\n",
" <td>0.17</td>\n",
" <td>6.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1829</th>\n",
" <td>3</td>\n",
" <td>768</td>\n",
" <td>RIGHT</td>\n",
" <td>0.26</td>\n",
" <td>1.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>551</th>\n",
" <td>3</td>\n",
" <td>619</td>\n",
" <td>LEFT</td>\n",
" <td>0.25</td>\n",
" <td>2.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>53</th>\n",
" <td>3</td>\n",
" <td>257</td>\n",
" <td>RIGHT</td>\n",
" <td>0.11</td>\n",
" <td>3.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>574</th>\n",
" <td>3</td>\n",
" <td>540</td>\n",
" <td>RIGHT</td>\n",
" <td>0.04</td>\n",
" <td>4.0</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" subject_id time (ms) response accuracy accuracy_rank\n",
"413 1 471 LEFT 0.80 1.0\n",
"1602 1 43 RIGHT 0.65 2.0\n",
"902 1 1093 LEFT 0.34 3.0\n",
"785 1 121 LEFT 0.10 4.0\n",
"1190 2 552 LEFT 0.43 1.0\n",
"1895 2 1036 LEFT 0.36 2.0\n",
"1393 2 903 RIGHT 0.33 3.0\n",
"158 2 743 RIGHT 0.32 4.0\n",
"1486 2 3 RIGHT 0.29 5.0\n",
"629 2 353 LEFT 0.17 6.0\n",
"1829 3 768 RIGHT 0.26 1.0\n",
"551 3 619 LEFT 0.25 2.0\n",
"53 3 257 RIGHT 0.11 3.0\n",
"574 3 540 RIGHT 0.04 4.0"
]
},
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"df.sort_values(['subject_id', 'accuracy_rank'])"
]
},
{
"cell_type": "markdown",
"id": "f57d47c8",
"metadata": {},
"source": [
"# In many cases, a window functions is combined with a sorting operation\n",
"\n",
"For example: for each subject, count the number of \"LEFT\" responses up until any moment in the experiment"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "f032f5db",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>subject_id</th>\n",
" <th>time (ms)</th>\n",
" <th>response</th>\n",
" <th>accuracy</th>\n",
" <th>accuracy_rank</th>\n",
" <th>is_left</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>574</th>\n",
" <td>3</td>\n",
" <td>540</td>\n",
" <td>RIGHT</td>\n",
" <td>0.04</td>\n",
" <td>4.0</td>\n",
" <td>False</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1190</th>\n",
" <td>2</td>\n",
" <td>552</td>\n",
" <td>LEFT</td>\n",
" <td>0.43</td>\n",
" <td>1.0</td>\n",
" <td>True</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1895</th>\n",
" <td>2</td>\n",
" <td>1036</td>\n",
" <td>LEFT</td>\n",
" <td>0.36</td>\n",
" <td>2.0</td>\n",
" <td>True</td>\n",
" </tr>\n",
" <tr>\n",
" <th>53</th>\n",
" <td>3</td>\n",
" <td>257</td>\n",
" <td>RIGHT</td>\n",
" <td>0.11</td>\n",
" <td>3.0</td>\n",
" <td>False</td>\n",
" </tr>\n",
" <tr>\n",
" <th>158</th>\n",
" <td>2</td>\n",
" <td>743</td>\n",
" <td>RIGHT</td>\n",
" <td>0.32</td>\n",
" <td>4.0</td>\n",
" <td>False</td>\n",
" </tr>\n",
" <tr>\n",
" <th>551</th>\n",
" <td>3</td>\n",
" <td>619</td>\n",
" <td>LEFT</td>\n",
" <td>0.25</td>\n",
" <td>2.0</td>\n",
" <td>True</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1602</th>\n",
" <td>1</td>\n",
" <td>43</td>\n",
" <td>RIGHT</td>\n",
" <td>0.65</td>\n",
" <td>2.0</td>\n",
" <td>False</td>\n",
" </tr>\n",
" <tr>\n",
" <th>413</th>\n",
" <td>1</td>\n",
" <td>471</td>\n",
" <td>LEFT</td>\n",
" <td>0.80</td>\n",
" <td>1.0</td>\n",
" <td>True</td>\n",
" </tr>\n",
" <tr>\n",
" <th>785</th>\n",
" <td>1</td>\n",
" <td>121</td>\n",
" <td>LEFT</td>\n",
" <td>0.10</td>\n",
" <td>4.0</td>\n",
" <td>True</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1393</th>\n",
" <td>2</td>\n",
" <td>903</td>\n",
" <td>RIGHT</td>\n",
" <td>0.33</td>\n",
" <td>3.0</td>\n",
" <td>False</td>\n",
" </tr>\n",
" <tr>\n",
" <th>629</th>\n",
" <td>2</td>\n",
" <td>353</td>\n",
" <td>LEFT</td>\n",
" <td>0.17</td>\n",
" <td>6.0</td>\n",
" <td>True</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1829</th>\n",
" <td>3</td>\n",
" <td>768</td>\n",
" <td>RIGHT</td>\n",
" <td>0.26</td>\n",
" <td>1.0</td>\n",
" <td>False</td>\n",
" </tr>\n",
" <tr>\n",
" <th>902</th>\n",
" <td>1</td>\n",
" <td>1093</td>\n",
" <td>LEFT</td>\n",
" <td>0.34</td>\n",
" <td>3.0</td>\n",
" <td>True</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1486</th>\n",
" <td>2</td>\n",
" <td>3</td>\n",
" <td>RIGHT</td>\n",
" <td>0.29</td>\n",
" <td>5.0</td>\n",
" <td>False</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" subject_id time (ms) response accuracy accuracy_rank is_left\n",
"574 3 540 RIGHT 0.04 4.0 False\n",
"1190 2 552 LEFT 0.43 1.0 True\n",
"1895 2 1036 LEFT 0.36 2.0 True\n",
"53 3 257 RIGHT 0.11 3.0 False\n",
"158 2 743 RIGHT 0.32 4.0 False\n",
"551 3 619 LEFT 0.25 2.0 True\n",
"1602 1 43 RIGHT 0.65 2.0 False\n",
"413 1 471 LEFT 0.80 1.0 True\n",
"785 1 121 LEFT 0.10 4.0 True\n",
"1393 2 903 RIGHT 0.33 3.0 False\n",
"629 2 353 LEFT 0.17 6.0 True\n",
"1829 3 768 RIGHT 0.26 1.0 False\n",
"902 1 1093 LEFT 0.34 3.0 True\n",
"1486 2 3 RIGHT 0.29 5.0 False"
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# Add a flag column \"is_left\", so that we can count the number of LEFT reponses using a cumulative sum\n",
"df['is_left'] = df['response'] == 'LEFT'\n",
"df"
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "f9420c3d",
"metadata": {
"scrolled": false
},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>subject_id</th>\n",
" <th>time (ms)</th>\n",
" <th>response</th>\n",
" <th>accuracy</th>\n",
" <th>accuracy_rank</th>\n",
" <th>is_left</th>\n",
" <th>nr_lefts</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>1602</th>\n",
" <td>1</td>\n",
" <td>43</td>\n",
" <td>RIGHT</td>\n",
" <td>0.65</td>\n",
" <td>2.0</td>\n",
" <td>False</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>413</th>\n",
" <td>1</td>\n",
" <td>471</td>\n",
" <td>LEFT</td>\n",
" <td>0.80</td>\n",
" <td>1.0</td>\n",
" <td>True</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>785</th>\n",
" <td>1</td>\n",
" <td>121</td>\n",
" <td>LEFT</td>\n",
" <td>0.10</td>\n",
" <td>4.0</td>\n",
" <td>True</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>902</th>\n",
" <td>1</td>\n",
" <td>1093</td>\n",
" <td>LEFT</td>\n",
" <td>0.34</td>\n",
" <td>3.0</td>\n",
" <td>True</td>\n",
" <td>3</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1190</th>\n",
" <td>2</td>\n",
" <td>552</td>\n",
" <td>LEFT</td>\n",
" <td>0.43</td>\n",
" <td>1.0</td>\n",
" <td>True</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1895</th>\n",
" <td>2</td>\n",
" <td>1036</td>\n",
" <td>LEFT</td>\n",
" <td>0.36</td>\n",
" <td>2.0</td>\n",
" <td>True</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>158</th>\n",
" <td>2</td>\n",
" <td>743</td>\n",
" <td>RIGHT</td>\n",
" <td>0.32</td>\n",
" <td>4.0</td>\n",
" <td>False</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1393</th>\n",
" <td>2</td>\n",
" <td>903</td>\n",
" <td>RIGHT</td>\n",
" <td>0.33</td>\n",
" <td>3.0</td>\n",
" <td>False</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>629</th>\n",
" <td>2</td>\n",
" <td>353</td>\n",
" <td>LEFT</td>\n",
" <td>0.17</td>\n",
" <td>6.0</td>\n",
" <td>True</td>\n",
" <td>3</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1486</th>\n",
" <td>2</td>\n",
" <td>3</td>\n",
" <td>RIGHT</td>\n",
" <td>0.29</td>\n",
" <td>5.0</td>\n",
" <td>False</td>\n",
" <td>3</td>\n",
" </tr>\n",
" <tr>\n",
" <th>574</th>\n",
" <td>3</td>\n",
" <td>540</td>\n",
" <td>RIGHT</td>\n",
" <td>0.04</td>\n",
" <td>4.0</td>\n",
" <td>False</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>53</th>\n",
" <td>3</td>\n",
" <td>257</td>\n",
" <td>RIGHT</td>\n",
" <td>0.11</td>\n",
" <td>3.0</td>\n",
" <td>False</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>551</th>\n",
" <td>3</td>\n",
" <td>619</td>\n",
" <td>LEFT</td>\n",
" <td>0.25</td>\n",
" <td>2.0</td>\n",
" <td>True</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1829</th>\n",
" <td>3</td>\n",
" <td>768</td>\n",
" <td>RIGHT</td>\n",
" <td>0.26</td>\n",
" <td>1.0</td>\n",
" <td>False</td>\n",
" <td>1</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" subject_id time (ms) response accuracy accuracy_rank is_left \\\n",
"1602 1 43 RIGHT 0.65 2.0 False \n",
"413 1 471 LEFT 0.80 1.0 True \n",
"785 1 121 LEFT 0.10 4.0 True \n",
"902 1 1093 LEFT 0.34 3.0 True \n",
"1190 2 552 LEFT 0.43 1.0 True \n",
"1895 2 1036 LEFT 0.36 2.0 True \n",
"158 2 743 RIGHT 0.32 4.0 False \n",
"1393 2 903 RIGHT 0.33 3.0 False \n",
"629 2 353 LEFT 0.17 6.0 True \n",
"1486 2 3 RIGHT 0.29 5.0 False \n",
"574 3 540 RIGHT 0.04 4.0 False \n",
"53 3 257 RIGHT 0.11 3.0 False \n",
"551 3 619 LEFT 0.25 2.0 True \n",
"1829 3 768 RIGHT 0.26 1.0 False \n",
"\n",
" nr_lefts \n",
"1602 0 \n",
"413 1 \n",
"785 2 \n",
"902 3 \n",
"1190 1 \n",
"1895 2 \n",
"158 2 \n",
"1393 2 \n",
"629 3 \n",
"1486 3 \n",
"574 0 \n",
"53 0 \n",
"551 1 \n",
"1829 1 "
]
},
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# Without sorting, we get the number of LEFT responses... in no particular order\n",
"df['nr_lefts'] = df.groupby('subject_id')['is_left'].cumsum()\n",
"df.sort_values(['subject_id'])"
]
},
{
"cell_type": "markdown",
"id": "ca1e8032",
"metadata": {},
"source": [
"# Window functions are also useful to compute changes in the data for each group\n",
"\n",
"In this case, the window function often uses the `shift(n)` method that lags the data by `n` rows"
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "18f440d3",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>subject_id</th>\n",
" <th>time (ms)</th>\n",
" <th>shifted time</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>1602</th>\n",
" <td>1</td>\n",
" <td>43</td>\n",
" <td>NaN</td>\n",
" </tr>\n",
" <tr>\n",
" <th>785</th>\n",
" <td>1</td>\n",
" <td>121</td>\n",
" <td>43.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>413</th>\n",
" <td>1</td>\n",
" <td>471</td>\n",
" <td>121.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>902</th>\n",
" <td>1</td>\n",
" <td>1093</td>\n",
" <td>471.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1486</th>\n",
" <td>2</td>\n",
" <td>3</td>\n",
" <td>NaN</td>\n",
" </tr>\n",
" <tr>\n",
" <th>629</th>\n",
" <td>2</td>\n",
" <td>353</td>\n",
" <td>3.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1190</th>\n",
" <td>2</td>\n",
" <td>552</td>\n",
" <td>353.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>158</th>\n",
" <td>2</td>\n",
" <td>743</td>\n",
" <td>552.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1393</th>\n",
" <td>2</td>\n",
" <td>903</td>\n",
" <td>743.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1895</th>\n",
" <td>2</td>\n",
" <td>1036</td>\n",
" <td>903.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>53</th>\n",
" <td>3</td>\n",
" <td>257</td>\n",
" <td>NaN</td>\n",
" </tr>\n",
" <tr>\n",
" <th>574</th>\n",
" <td>3</td>\n",
" <td>540</td>\n",
" <td>257.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>551</th>\n",
" <td>3</td>\n",
" <td>619</td>\n",
" <td>540.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1829</th>\n",
" <td>3</td>\n",
" <td>768</td>\n",
" <td>619.0</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" subject_id time (ms) shifted time\n",
"1602 1 43 NaN\n",
"785 1 121 43.0\n",
"413 1 471 121.0\n",
"902 1 1093 471.0\n",
"1486 2 3 NaN\n",
"629 2 353 3.0\n",
"1190 2 552 353.0\n",
"158 2 743 552.0\n",
"1393 2 903 743.0\n",
"1895 2 1036 903.0\n",
"53 3 257 NaN\n",
"574 3 540 257.0\n",
"551 3 619 540.0\n",
"1829 3 768 619.0"
]
},
"execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"df['shifted time'] = (\n",
" df\n",
" .sort_values('time (ms)')\n",
" .groupby('subject_id')['time (ms)']\n",
" .shift(1)\n",
")\n",
"df.sort_values(['subject_id', 'time (ms)'])[['subject_id', 'time (ms)', 'shifted time']]"
]
},
{
"cell_type": "code",
"execution_count": 11,
"id": "4f1cb393",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>subject_id</th>\n",
" <th>time (ms)</th>\n",
" <th>time from prev</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>1602</th>\n",
" <td>1</td>\n",
" <td>43</td>\n",
" <td>NaN</td>\n",
" </tr>\n",
" <tr>\n",
" <th>785</th>\n",
" <td>1</td>\n",
" <td>121</td>\n",
" <td>78.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>413</th>\n",
" <td>1</td>\n",
" <td>471</td>\n",
" <td>350.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>902</th>\n",
" <td>1</td>\n",
" <td>1093</td>\n",
" <td>622.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1486</th>\n",
" <td>2</td>\n",
" <td>3</td>\n",
" <td>NaN</td>\n",
" </tr>\n",
" <tr>\n",
" <th>629</th>\n",
" <td>2</td>\n",
" <td>353</td>\n",
" <td>350.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1190</th>\n",
" <td>2</td>\n",
" <td>552</td>\n",
" <td>199.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>158</th>\n",
" <td>2</td>\n",
" <td>743</td>\n",
" <td>191.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1393</th>\n",
" <td>2</td>\n",
" <td>903</td>\n",
" <td>160.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1895</th>\n",
" <td>2</td>\n",
" <td>1036</td>\n",
" <td>133.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>53</th>\n",
" <td>3</td>\n",
" <td>257</td>\n",
" <td>NaN</td>\n",
" </tr>\n",
" <tr>\n",
" <th>574</th>\n",
" <td>3</td>\n",
" <td>540</td>\n",
" <td>283.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>551</th>\n",
" <td>3</td>\n",
" <td>619</td>\n",
" <td>79.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1829</th>\n",
" <td>3</td>\n",
" <td>768</td>\n",
" <td>149.0</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" subject_id time (ms) time from prev\n",
"1602 1 43 NaN\n",
"785 1 121 78.0\n",
"413 1 471 350.0\n",
"902 1 1093 622.0\n",
"1486 2 3 NaN\n",
"629 2 353 350.0\n",
"1190 2 552 199.0\n",
"158 2 743 191.0\n",
"1393 2 903 160.0\n",
"1895 2 1036 133.0\n",
"53 3 257 NaN\n",
"574 3 540 283.0\n",
"551 3 619 79.0\n",
"1829 3 768 149.0"
]
},
"execution_count": 11,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"df['time from prev'] = df['time (ms)'] - df['shifted time']\n",
"df.sort_values(['subject_id', 'time (ms)'])[['subject_id', 'time (ms)', 'time from prev']]"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "3d06a890",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "87a52b7c",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "61badaf1",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.3"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Reference in a new issue Copy permalink