{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Visualizing Benchmarks"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Preparation\n",
"We start from loading the necessary modules and the data, which is immediately converted to dataframes.\n",
"\n",
"Then, we process the data if necessary (usually by converting it into long format, which is compatible with Altair)."
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"import yaml\n",
"import altair as alt\n",
"from altair_saver import save\n",
"import pandas as pd"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"chart_dir = 'charts'\n",
"with open(\"results.yaml\", 'r') as stream:\n",
" data = yaml.safe_load(stream)\n",
"if not os.path.exists(chart_dir):\n",
" os.mkdir(chart_dir)"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"external, internal = data['external'], data['internal']\n",
"ext_dset_iter = pd.DataFrame(external['dataset_iteration'])\n",
"ext_seq_acc = pd.DataFrame(external['sequential_access'])\n",
"int_compres = pd.DataFrame(internal['compression'])\n",
"int_rand_acc = pd.DataFrame(internal['random_access'])\n",
"int_dset_iter = pd.DataFrame(internal['dataset_iteration'])"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"stacked_ext_dset_iter = ext_dset_iter.set_index('loader').stack().reset_index().rename(columns={'level_1': 'Dataset', 0: 'time'})\n",
"melt_int_rand_acc = int_rand_acc.melt('batch_size', var_name='Dataset', value_name='Time')"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Drawing"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"graph_ext_dset_iter = alt.Chart(stacked_ext_dset_iter).mark_bar(\n",
" cornerRadiusBottomRight=2,\n",
" cornerRadiusTopRight=2\n",
").encode(\n",
" x=alt.X('time:Q', title=\"Time in seconds (log scale)\", scale=alt.Scale(type='log'), axis=alt.Axis(grid=False)),\n",
" y=alt.Y('Dataset:O', axis=None),\n",
" color='Dataset:N',\n",
" row=alt.Row('loader:N', header=alt.Header(title=None, labelOrient='top'))\n",
").properties(\n",
" height=alt.Step(25),\n",
" width=1000,\n",
" title=\"Dataset Iteration (external)\"\n",
").configure_view(\n",
" stroke='transparent'\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"wr = alt.Chart(ext_seq_acc).mark_rect()\n",
"wr_x = alt.X('dataset:O', title=None)\n",
"wr_y = alt.Y('framework:O', title=None)\n",
"wr_props = {'height': alt.Step(25), 'width': 100}\n",
"read = wr.encode(\n",
" y=wr_y,\n",
" x=wr_x,\n",
" color='read:Q'\n",
")\n",
"write = wr.encode(\n",
" y=wr_y,\n",
" x=wr_x,\n",
" color='write:Q'\n",
")\n",
"graph_ext_seq_acc = (read.properties(**wr_props, title='Read') | write.properties(**wr_props, title='Write')).properties(title=\"Sequential Access\")"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
"graph_int_rand_acc = alt.Chart(melt_int_rand_acc).mark_line().encode(\n",
" x=\"batch_size\",\n",
" y=\"Time:Q\",\n",
" color='Dataset:N'\n",
").properties(title=\"Random Access\")"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
"graph_int_dset_iter = alt.Chart(int_dset_iter).mark_circle().encode(\n",
" y=alt.Y(\"type\", title=None),\n",
" x=alt.X(\"time:Q\", axis=alt.Axis(grid=False), title=\"Time in seconds\"),\n",
" color='prefetch_factor:N',\n",
" size=alt.Size('batch_size:Q', scale=alt.Scale(type='log'))\n",
").properties(title=\"Dataset Iteration (internal)\", height=170).configure_view(strokeOpacity=0)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Graphs"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
"\n",
""
],
"text/plain": [
"alt.Chart(...)"
]
},
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"graph_ext_dset_iter"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
"\n",
""
],
"text/plain": [
"alt.HConcatChart(...)"
]
},
"execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"graph_ext_seq_acc"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
"\n",
""
],
"text/plain": [
"alt.Chart(...)"
]
},
"execution_count": 11,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"graph_int_rand_acc"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
"\n",
""
],
"text/plain": [
"alt.Chart(...)"
]
},
"execution_count": 12,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"graph_int_dset_iter"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Saving Graphs"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [],
"source": [
"for graph in graph_ext_dset_iter, graph_ext_seq_acc, graph_int_rand_acc, graph_int_dset_iter:\n",
" save(graph, f\"charts/{graph.title.replace(' ', '_')}.png\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.9"
}
},
"nbformat": 4,
"nbformat_minor": 4
}