Commit eb590eea authored by Lukas-Stingl's avatar Lukas-Stingl
Browse files

import function

parent c4fadd88
{ {
"metadata": { "cells": [
"language_info": { {
"codemirror_mode": { "cell_type": "code",
"name": "ipython", "execution_count": 3,
"version": 3 "metadata": {},
}, "outputs": [],
"file_extension": ".py", "source": [
"mimetype": "text/x-python", "import pandas as pd\n",
"name": "python", "import json\n",
"nbconvert_exporter": "python", "from pandas.io.json import json_normalize\n",
"pygments_lexer": "ipython3", "import os"
"version": "3.9.5" ]
}, },
"orig_nbformat": 4, {
"kernelspec": { "cell_type": "code",
"name": "python3", "execution_count": 4,
"display_name": "Python 3.9.5 64-bit ('bda': conda)" "metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>id</th>\n",
" <th>deveui</th>\n",
" <th>unix_time</th>\n",
" <th>client_id</th>\n",
" <th>created_at</th>\n",
" <th>decoded_data</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>4110164</td>\n",
" <td>70B3D500700016DF</td>\n",
" <td>1620495074723</td>\n",
" <td>FES Receiver</td>\n",
" <td>2021-05-08 19:31:14.932784</td>\n",
" <td>{'sensor_data': {'Status': ['SS', 'Standalone'...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>4109667</td>\n",
" <td>70B3D500700016DF</td>\n",
" <td>1620491472910</td>\n",
" <td>FES Receiver</td>\n",
" <td>2021-05-08 18:31:13.354255</td>\n",
" <td>{'sensor_data': {'Status': ['SS', 'Standalone'...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>4107761</td>\n",
" <td>70B3D500700016DF</td>\n",
" <td>1620469874104</td>\n",
" <td>FES Receiver</td>\n",
" <td>2021-05-08 12:31:14.486747</td>\n",
" <td>{'sensor_data': {'Status': ['SS', 'Standalone'...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>4107253</td>\n",
" <td>70B3D500700016DF</td>\n",
" <td>1620466274295</td>\n",
" <td>FES Receiver</td>\n",
" <td>2021-05-08 11:31:14.667904</td>\n",
" <td>{'sensor_data': {'Status': ['SS', 'Standalone'...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>4106117</td>\n",
" <td>70B3D500700016DF</td>\n",
" <td>1620455474784</td>\n",
" <td>FES Receiver</td>\n",
" <td>2021-05-08 08:31:15.128783</td>\n",
" <td>{'sensor_data': {'Status': ['SS', 'Standalone'...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3278</th>\n",
" <td>755012</td>\n",
" <td>70B3D500700016DF</td>\n",
" <td>1591275152402</td>\n",
" <td>FES Receiver</td>\n",
" <td>2020-06-04 14:52:32.484604</td>\n",
" <td>{'sensor_data': {'Status': ['SS', 'Standalone'...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3279</th>\n",
" <td>754786</td>\n",
" <td>70B3D500700016DF</td>\n",
" <td>1591271552608</td>\n",
" <td>FES Receiver</td>\n",
" <td>2020-06-04 13:52:32.695082</td>\n",
" <td>{'sensor_data': {'Status': ['SS', 'Standalone'...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3280</th>\n",
" <td>754545</td>\n",
" <td>70B3D500700016DF</td>\n",
" <td>1591267952759</td>\n",
" <td>FES Receiver</td>\n",
" <td>2020-06-04 12:52:32.850809</td>\n",
" <td>{'sensor_data': {'Status': ['SS', 'Standalone'...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3281</th>\n",
" <td>753761</td>\n",
" <td>70B3D500700016DF</td>\n",
" <td>1591257153525</td>\n",
" <td>FES Receiver</td>\n",
" <td>2020-06-04 09:52:33.632266</td>\n",
" <td>{'sensor_data': {'Status': ['SS', 'Standalone'...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3282</th>\n",
" <td>753560</td>\n",
" <td>70B3D500700016DF</td>\n",
" <td>1591253853989</td>\n",
" <td>FES Receiver</td>\n",
" <td>2020-06-04 08:57:34.099067</td>\n",
" <td>{'sensor_data': {'Status': ['SS', 'Standalone'...</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>3283 rows × 6 columns</p>\n",
"</div>"
],
"text/plain": [
" id deveui unix_time client_id \\\n",
"0 4110164 70B3D500700016DF 1620495074723 FES Receiver \n",
"1 4109667 70B3D500700016DF 1620491472910 FES Receiver \n",
"2 4107761 70B3D500700016DF 1620469874104 FES Receiver \n",
"3 4107253 70B3D500700016DF 1620466274295 FES Receiver \n",
"4 4106117 70B3D500700016DF 1620455474784 FES Receiver \n",
"... ... ... ... ... \n",
"3278 755012 70B3D500700016DF 1591275152402 FES Receiver \n",
"3279 754786 70B3D500700016DF 1591271552608 FES Receiver \n",
"3280 754545 70B3D500700016DF 1591267952759 FES Receiver \n",
"3281 753761 70B3D500700016DF 1591257153525 FES Receiver \n",
"3282 753560 70B3D500700016DF 1591253853989 FES Receiver \n",
"\n",
" created_at \\\n",
"0 2021-05-08 19:31:14.932784 \n",
"1 2021-05-08 18:31:13.354255 \n",
"2 2021-05-08 12:31:14.486747 \n",
"3 2021-05-08 11:31:14.667904 \n",
"4 2021-05-08 08:31:15.128783 \n",
"... ... \n",
"3278 2020-06-04 14:52:32.484604 \n",
"3279 2020-06-04 13:52:32.695082 \n",
"3280 2020-06-04 12:52:32.850809 \n",
"3281 2020-06-04 09:52:33.632266 \n",
"3282 2020-06-04 08:57:34.099067 \n",
"\n",
" decoded_data \n",
"0 {'sensor_data': {'Status': ['SS', 'Standalone'... \n",
"1 {'sensor_data': {'Status': ['SS', 'Standalone'... \n",
"2 {'sensor_data': {'Status': ['SS', 'Standalone'... \n",
"3 {'sensor_data': {'Status': ['SS', 'Standalone'... \n",
"4 {'sensor_data': {'Status': ['SS', 'Standalone'... \n",
"... ... \n",
"3278 {'sensor_data': {'Status': ['SS', 'Standalone'... \n",
"3279 {'sensor_data': {'Status': ['SS', 'Standalone'... \n",
"3280 {'sensor_data': {'Status': ['SS', 'Standalone'... \n",
"3281 {'sensor_data': {'Status': ['SS', 'Standalone'... \n",
"3282 {'sensor_data': {'Status': ['SS', 'Standalone'... \n",
"\n",
"[3283 rows x 6 columns]"
]
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"\n",
"df = pd.read_json('../data/raw/data/70b3d500700016dF.txt', lines=True)\n",
"\n",
"\n",
"\n",
"df_garbage = pd.DataFrame(df[1][0])\n",
"df_garbage"
]
}, },
"interpreter": {
"hash": "f8505f590489ae640872df7de7f2cd829a99b5a918ef43d73c16b5853871bccb"
}
},
"nbformat": 4,
"nbformat_minor": 2,
"cells": [
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 1, "execution_count": 5,
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
"output_type": "error", "data": {
"ename": "ModuleNotFoundError", "text/html": [
"evalue": "No module named 'pandas'", "<div>\n",
"traceback": [ "<style scoped>\n",
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", " .dataframe tbody tr th:only-of-type {\n",
"\u001b[0;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)", " vertical-align: middle;\n",
"\u001b[0;32m<ipython-input-1-7dd3504c366f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0mpandas\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", " }\n",
"\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'pandas'" "\n",
] " .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>id</th>\n",
" <th>deveui</th>\n",
" <th>unix_time</th>\n",
" <th>client_id</th>\n",
" <th>created_at</th>\n",
" <th>decoded_data</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>4110576</td>\n",
" <td>70B3D500700016DA</td>\n",
" <td>1620498360377</td>\n",
" <td>FES Receiver</td>\n",
" <td>2021-05-08 20:26:00.596017</td>\n",
" <td>{'sensor_data': {'Status': ['SS', 'Standalone'...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>4110093</td>\n",
" <td>70B3D500700016DA</td>\n",
" <td>1620494765501</td>\n",
" <td>FES Receiver</td>\n",
" <td>2021-05-08 19:26:05.713448</td>\n",
" <td>{'sensor_data': {'Status': ['SS', 'Standalone'...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>4109607</td>\n",
" <td>70B3D500700016DA</td>\n",
" <td>1620491158880</td>\n",
" <td>FES Receiver</td>\n",
" <td>2021-05-08 18:25:59.323267</td>\n",
" <td>{'sensor_data': {'Status': ['SS', 'Standalone'...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>4109118</td>\n",
" <td>70B3D500700016DA</td>\n",
" <td>1620487559091</td>\n",
" <td>FES Receiver</td>\n",
" <td>2021-05-08 17:25:59.523137</td>\n",
" <td>{'sensor_data': {'Status': ['SS', 'Standalone'...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>4108627</td>\n",
" <td>70B3D500700016DA</td>\n",
" <td>1620483959154</td>\n",
" <td>FES Receiver</td>\n",
" <td>2021-05-08 16:25:59.577446</td>\n",
" <td>{'sensor_data': {'Status': ['SS', 'Standalone'...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>515032</th>\n",
" <td>598759</td>\n",
" <td>70B3D50070001789</td>\n",
" <td>1588992614891</td>\n",
" <td>hi3</td>\n",
" <td>2020-05-09 04:50:15.056761</td>\n",
" <td>{'sensor_data': {'Status': ['SS', 'Standalone'...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>515033</th>\n",
" <td>598510</td>\n",
" <td>70B3D50070001789</td>\n",
" <td>1588989015087</td>\n",
" <td>hi3</td>\n",
" <td>2020-05-09 03:50:15.256986</td>\n",
" <td>{'sensor_data': {'Status': ['SS', 'Standalone'...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>515034</th>\n",
" <td>598261</td>\n",
" <td>70B3D50070001789</td>\n",
" <td>1588985415234</td>\n",
" <td>hi3</td>\n",
" <td>2020-05-09 02:50:15.408009</td>\n",
" <td>{'sensor_data': {'Status': ['SS', 'Standalone'...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>515035</th>\n",
" <td>598017</td>\n",
" <td>70B3D50070001789</td>\n",
" <td>1588981815380</td>\n",
" <td>hi3</td>\n",
" <td>2020-05-09 01:50:15.558981</td>\n",
" <td>{'sensor_data': {'Status': ['SS', 'Standalone'...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>515036</th>\n",
" <td>597775</td>\n",
" <td>70B3D50070001789</td>\n",
" <td>1588978215547</td>\n",
" <td>hi3</td>\n",
" <td>2020-05-09 00:50:15.726339</td>\n",
" <td>{'sensor_data': {'Status': ['SS', 'Standalone'...</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>515037 rows × 6 columns</p>\n",
"</div>"
],
"text/plain": [
" id deveui unix_time client_id \\\n",
"0 4110576 70B3D500700016DA 1620498360377 FES Receiver \n",
"1 4110093 70B3D500700016DA 1620494765501 FES Receiver \n",
"2 4109607 70B3D500700016DA 1620491158880 FES Receiver \n",
"3 4109118 70B3D500700016DA 1620487559091 FES Receiver \n",
"4 4108627 70B3D500700016DA 1620483959154 FES Receiver \n",
"... ... ... ... ... \n",
"515032 598759 70B3D50070001789 1588992614891 hi3 \n",
"515033 598510 70B3D50070001789 1588989015087 hi3 \n",
"515034 598261 70B3D50070001789 1588985415234 hi3 \n",
"515035 598017 70B3D50070001789 1588981815380 hi3 \n",
"515036 597775 70B3D50070001789 1588978215547 hi3 \n",
"\n",
" created_at \\\n",
"0 2021-05-08 20:26:00.596017 \n",
"1 2021-05-08 19:26:05.713448 \n",
"2 2021-05-08 18:25:59.323267 \n",
"3 2021-05-08 17:25:59.523137 \n",
"4 2021-05-08 16:25:59.577446 \n",
"... ... \n",
"515032 2020-05-09 04:50:15.056761 \n",
"515033 2020-05-09 03:50:15.256986 \n",
"515034 2020-05-09 02:50:15.408009 \n",
"515035 2020-05-09 01:50:15.558981 \n",
"515036 2020-05-09 00:50:15.726339 \n",
"\n",
" decoded_data \n",
"0 {'sensor_data': {'Status': ['SS', 'Standalone'... \n",
"1 {'sensor_data': {'Status': ['SS', 'Standalone'... \n",
"2 {'sensor_data': {'Status': ['SS', 'Standalone'... \n",
"3 {'sensor_data': {'Status': ['SS', 'Standalone'... \n",
"4 {'sensor_data': {'Status': ['SS', 'Standalone'... \n",
"... ... \n",
"515032 {'sensor_data': {'Status': ['SS', 'Standalone'... \n",
"515033 {'sensor_data': {'Status': ['SS', 'Standalone'... \n",
"515034 {'sensor_data': {'Status': ['SS', 'Standalone'... \n",
"515035 {'sensor_data': {'Status': ['SS', 'Standalone'... \n",
"515036 {'sensor_data': {'Status': ['SS', 'Standalone'... \n",
"\n",
"[515037 rows x 6 columns]"
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
} }
], ],
"source": [ "source": [
"import pandas as pd" "\n",
"# df = pd.read_json('../data/raw/data/70b3d500700016dA.txt', lines=True)\n",
"temp = pd.DataFrame()\n",
"path_to_json = '../data/raw/data/' \n",
"json_files = [pos_json for pos_json in os.listdir(path_to_json) if pos_json.endswith('.txt')]\n",
"\n",
"\n",
"\n",
"dfs = []\n",
"\n",
"for filename in json_files:\n",
" df = pd.read_json(path_to_json + filename, lines=True)\n",
" df_garbage = pd.DataFrame(df[1][0])\n",
" dfs.append(df_garbage)\n",
" \n",
"temp = pd.concat(dfs,ignore_index=True)\n",
"temp\n"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 4,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [] "source": [
"# df.to_csv('..\\test.csv', index = False)"
]
} }
] ],
} "metadata": {
\ No newline at end of file "interpreter": {
"hash": "f8505f590489ae640872df7de7f2cd829a99b5a918ef43d73c16b5853871bccb"
},
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.0"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` ``` python
import pandas as pd import pandas as pd
import json
from pandas.io.json import json_normalize
import os
``` ```
%%%% Output: error %% Cell type:code id: tags:
``` python
df = pd.read_json('../data/raw/data/70b3d500700016dF.txt', lines=True)
df_garbage = pd.DataFrame(df[1][0])
df_garbage
```
%%%% Output: execute_result
id deveui unix_time client_id \
0 4110164 70B3D500700016DF 1620495074723 FES Receiver
1 4109667 70B3D500700016DF 1620491472910 FES Receiver
2 4107761 70B3D500700016DF 1620469874104 FES Receiver
3 4107253 70B3D500700016DF 1620466274295 FES Receiver
4 4106117 70B3D500700016DF 1620455474784 FES Receiver
... ... ... ... ...
3278 755012 70B3D500700016DF 1591275152402 FES Receiver
3279 754786 70B3D500700016DF 1591271552608 FES Receiver
3280 754545 70B3D500700016DF 1591267952759 FES Receiver
3281 753761 70B3D500700016DF 1591257153525 FES Receiver
3282 753560 70B3D500700016DF 1591253853989 FES Receiver
created_at \
0 2021-05-08 19:31:14.932784
1 2021-05-08 18:31:13.354255
2 2021-05-08 12:31:14.486747
3 2021-05-08 11:31:14.667904
4 2021-05-08 08:31:15.128783
... ...
3278 2020-06-04 14:52:32.484604
3279 2020-06-04 13:52:32.695082
3280 2020-06-04 12:52:32.850809
3281 2020-06-04 09:52:33.632266
3282 2020-06-04 08:57:34.099067
decoded_data
0 {'sensor_data': {'Status': ['SS', 'Standalone'...
1 {'sensor_data': {'Status': ['SS', 'Standalone'...
2 {'sensor_data': {'Status': ['SS', 'Standalone'...
3 {'sensor_data': {'Status': ['SS', 'Standalone'...
4 {'sensor_data': {'Status': ['SS', 'Standalone'...
... ...
3278 {'sensor_data': {'Status': ['SS', 'Standalone'...
3279 {'sensor_data': {'Status': ['SS', 'Standalone'...
3280 {'sensor_data': {'Status': ['SS', 'Standalone'...
3281 {'sensor_data': {'Status': ['SS', 'Standalone'...
3282 {'sensor_data': {'Status': ['SS', 'Standalone'...
[3283 rows x 6 columns]
%% Cell type:code id: tags:
``` python
# df = pd.read_json('../data/raw/data/70b3d500700016dA.txt', lines=True)
temp = pd.DataFrame()
path_to_json = '../data/raw/data/'
json_files = [pos_json for pos_json in os.listdir(path_to_json) if pos_json.endswith('.txt')]
dfs = []
for filename in json_files:
df = pd.read_json(path_to_json + filename, lines=True)
df_garbage = pd.DataFrame(df[1][0])
dfs.append(df_garbage)
temp = pd.concat(dfs,ignore_index=True)
temp
```
--------------------------------------------------------------------------- %%%% Output: execute_result
ModuleNotFoundError Traceback (most recent call last)
<ipython-input-1-7dd3504c366f> in <module>
----> 1 import pandas as pd
ModuleNotFoundError: No module named 'pandas' id deveui unix_time client_id \
0 4110576 70B3D500700016DA 1620498360377 FES Receiver
1 4110093 70B3D500700016DA 1620494765501 FES Receiver
2 4109607 70B3D500700016DA 1620491158880 FES Receiver
3 4109118 70B3D500700016DA 1620487559091 FES Receiver
4 4108627 70B3D500700016DA 1620483959154 FES Receiver
... ... ... ... ...
515032 598759 70B3D50070001789 1588992614891 hi3
515033 598510 70B3D50070001789 1588989015087 hi3
515034 598261 70B3D50070001789 1588985415234 hi3
515035 598017 70B3D50070001789 1588981815380 hi3
515036 597775 70B3D50070001789 1588978215547 hi3
created_at \
0 2021-05-08 20:26:00.596017
1 2021-05-08 19:26:05.713448
2 2021-05-08 18:25:59.323267
3 2021-05-08 17:25:59.523137
4 2021-05-08 16:25:59.577446
... ...
515032 2020-05-09 04:50:15.056761
515033 2020-05-09 03:50:15.256986
515034 2020-05-09 02:50:15.408009
515035 2020-05-09 01:50:15.558981
515036 2020-05-09 00:50:15.726339
decoded_data
0 {'sensor_data': {'Status': ['SS', 'Standalone'...
1 {'sensor_data': {'Status': ['SS', 'Standalone'...
2 {'sensor_data': {'Status': ['SS', 'Standalone'...
3 {'sensor_data': {'Status': ['SS', 'Standalone'...
4 {'sensor_data': {'Status': ['SS', 'Standalone'...
... ...
515032 {'sensor_data': {'Status': ['SS', 'Standalone'...
515033 {'sensor_data': {'Status': ['SS', 'Standalone'...
515034 {'sensor_data': {'Status': ['SS', 'Standalone'...
515035 {'sensor_data': {'Status': ['SS', 'Standalone'...
515036 {'sensor_data': {'Status': ['SS', 'Standalone'...
[515037 rows x 6 columns]
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` ``` python
# df.to_csv('..\test.csv', index = False)
``` ```
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment