|
|
@ -47,7 +47,7 @@ |
|
|
}, |
|
|
}, |
|
|
{ |
|
|
{ |
|
|
"cell_type": "code", |
|
|
"cell_type": "code", |
|
|
"execution_count": 1, |
|
|
"execution_count": 34, |
|
|
"id": "ab6c6c81-6ac1-4668-a79b-a9a0341fb35a", |
|
|
"id": "ab6c6c81-6ac1-4668-a79b-a9a0341fb35a", |
|
|
"metadata": { |
|
|
"metadata": { |
|
|
"tags": [] |
|
|
"tags": [] |
|
|
@ -59,7 +59,7 @@ |
|
|
"False" |
|
|
"False" |
|
|
] |
|
|
] |
|
|
}, |
|
|
}, |
|
|
"execution_count": 1, |
|
|
"execution_count": 34, |
|
|
"metadata": {}, |
|
|
"metadata": {}, |
|
|
"output_type": "execute_result" |
|
|
"output_type": "execute_result" |
|
|
} |
|
|
} |
|
|
@ -71,12 +71,13 @@ |
|
|
"from datetime import datetime\n", |
|
|
"from datetime import datetime\n", |
|
|
"\n", |
|
|
"\n", |
|
|
"import duckdb\n", |
|
|
"import duckdb\n", |
|
|
"import influxdb_client\n", |
|
|
|
|
|
"import matplotlib.pyplot as plt\n", |
|
|
"import matplotlib.pyplot as plt\n", |
|
|
"import numpy as np\n", |
|
|
"import numpy as np\n", |
|
|
"import pandas as pd\n", |
|
|
"import pandas as pd\n", |
|
|
"from clickhouse_driver import Client\n", |
|
|
"from clickhouse_driver import Client\n", |
|
|
"from dotenv import load_dotenv\n", |
|
|
"from dotenv import load_dotenv\n", |
|
|
|
|
|
"from influxdb_client import InfluxDBClient\n", |
|
|
|
|
|
"from influxdb_client.client.write_api import SYNCHRONOUS\n", |
|
|
"from minio import Minio\n", |
|
|
"from minio import Minio\n", |
|
|
"from pymongo import MongoClient\n", |
|
|
"from pymongo import MongoClient\n", |
|
|
"from pytz import timezone\n", |
|
|
"from pytz import timezone\n", |
|
|
@ -100,7 +101,7 @@ |
|
|
}, |
|
|
}, |
|
|
{ |
|
|
{ |
|
|
"cell_type": "code", |
|
|
"cell_type": "code", |
|
|
"execution_count": 3, |
|
|
"execution_count": 42, |
|
|
"id": "968403e3-2e5e-4834-b969-be4600e2963a", |
|
|
"id": "968403e3-2e5e-4834-b969-be4600e2963a", |
|
|
"metadata": { |
|
|
"metadata": { |
|
|
"tags": [] |
|
|
"tags": [] |
|
|
@ -135,7 +136,7 @@ |
|
|
}, |
|
|
}, |
|
|
{ |
|
|
{ |
|
|
"cell_type": "code", |
|
|
"cell_type": "code", |
|
|
"execution_count": 5, |
|
|
"execution_count": 4, |
|
|
"id": "3634a4ec-04c2-4f1e-8659-5d22eb17a254", |
|
|
"id": "3634a4ec-04c2-4f1e-8659-5d22eb17a254", |
|
|
"metadata": {}, |
|
|
"metadata": {}, |
|
|
"outputs": [ |
|
|
"outputs": [ |
|
|
@ -252,7 +253,7 @@ |
|
|
"999999 2023-03-03 18:14:45 1.062535 1.062520 1.062520 1.062580 59 " |
|
|
"999999 2023-03-03 18:14:45 1.062535 1.062520 1.062520 1.062580 59 " |
|
|
] |
|
|
] |
|
|
}, |
|
|
}, |
|
|
"execution_count": 5, |
|
|
"execution_count": 4, |
|
|
"metadata": {}, |
|
|
"metadata": {}, |
|
|
"output_type": "execute_result" |
|
|
"output_type": "execute_result" |
|
|
} |
|
|
} |
|
|
@ -291,7 +292,7 @@ |
|
|
}, |
|
|
}, |
|
|
{ |
|
|
{ |
|
|
"cell_type": "code", |
|
|
"cell_type": "code", |
|
|
"execution_count": 6, |
|
|
"execution_count": 5, |
|
|
"id": "27de1ec8-4de1-440a-b555-b4a46c5ef7ce", |
|
|
"id": "27de1ec8-4de1-440a-b555-b4a46c5ef7ce", |
|
|
"metadata": {}, |
|
|
"metadata": {}, |
|
|
"outputs": [], |
|
|
"outputs": [], |
|
|
@ -305,6 +306,7 @@ |
|
|
"cell_type": "markdown", |
|
|
"cell_type": "markdown", |
|
|
"id": "4a8d5703-9bc9-4d38-83ff-457159304d58", |
|
|
"id": "4a8d5703-9bc9-4d38-83ff-457159304d58", |
|
|
"metadata": { |
|
|
"metadata": { |
|
|
|
|
|
"jp-MarkdownHeadingCollapsed": true, |
|
|
"tags": [] |
|
|
"tags": [] |
|
|
}, |
|
|
}, |
|
|
"source": [ |
|
|
"source": [ |
|
|
@ -313,7 +315,7 @@ |
|
|
}, |
|
|
}, |
|
|
{ |
|
|
{ |
|
|
"cell_type": "code", |
|
|
"cell_type": "code", |
|
|
"execution_count": 20, |
|
|
"execution_count": 8, |
|
|
"id": "c3202bbb-2655-45b2-b166-9f45a3ef854c", |
|
|
"id": "c3202bbb-2655-45b2-b166-9f45a3ef854c", |
|
|
"metadata": { |
|
|
"metadata": { |
|
|
"tags": [] |
|
|
"tags": [] |
|
|
@ -325,7 +327,7 @@ |
|
|
"'Database created'" |
|
|
"'Database created'" |
|
|
] |
|
|
] |
|
|
}, |
|
|
}, |
|
|
"execution_count": 20, |
|
|
"execution_count": 8, |
|
|
"metadata": {}, |
|
|
"metadata": {}, |
|
|
"output_type": "execute_result" |
|
|
"output_type": "execute_result" |
|
|
} |
|
|
} |
|
|
@ -379,7 +381,7 @@ |
|
|
}, |
|
|
}, |
|
|
{ |
|
|
{ |
|
|
"cell_type": "code", |
|
|
"cell_type": "code", |
|
|
"execution_count": 21, |
|
|
"execution_count": 9, |
|
|
"id": "cc4865b3-a1bc-4a35-9624-15334754b3a1", |
|
|
"id": "cc4865b3-a1bc-4a35-9624-15334754b3a1", |
|
|
"metadata": {}, |
|
|
"metadata": {}, |
|
|
"outputs": [], |
|
|
"outputs": [], |
|
|
@ -393,7 +395,7 @@ |
|
|
}, |
|
|
}, |
|
|
{ |
|
|
{ |
|
|
"cell_type": "code", |
|
|
"cell_type": "code", |
|
|
"execution_count": 22, |
|
|
"execution_count": 10, |
|
|
"id": "1fac82c1-2d04-44ef-893a-dc13b755e6d8", |
|
|
"id": "1fac82c1-2d04-44ef-893a-dc13b755e6d8", |
|
|
"metadata": {}, |
|
|
"metadata": {}, |
|
|
"outputs": [], |
|
|
"outputs": [], |
|
|
@ -407,7 +409,7 @@ |
|
|
}, |
|
|
}, |
|
|
{ |
|
|
{ |
|
|
"cell_type": "code", |
|
|
"cell_type": "code", |
|
|
"execution_count": 23, |
|
|
"execution_count": 11, |
|
|
"id": "597ae7bd-2eea-44d7-b379-f0eb7e745c15", |
|
|
"id": "597ae7bd-2eea-44d7-b379-f0eb7e745c15", |
|
|
"metadata": { |
|
|
"metadata": { |
|
|
"tags": [] |
|
|
"tags": [] |
|
|
@ -447,64 +449,64 @@ |
|
|
" </thead>\n", |
|
|
" </thead>\n", |
|
|
" <tbody>\n", |
|
|
" <tbody>\n", |
|
|
" <tr>\n", |
|
|
" <tr>\n", |
|
|
" <th>2999995</th>\n", |
|
|
" <th>999995</th>\n", |
|
|
" <td>8230798</td>\n", |
|
|
" <td>7857774</td>\n", |
|
|
" <td>2023-05-03 10:35:00</td>\n", |
|
|
" <td>2023-02-01 17:06:00</td>\n", |
|
|
" <td>1683110115000000000</td>\n", |
|
|
" <td>1675271175000000000</td>\n", |
|
|
" <td>2023-05-03 10:35:15</td>\n", |
|
|
" <td>2023-02-01 17:06:15</td>\n", |
|
|
" <td>1.103340</td>\n", |
|
|
" <td>1.091725</td>\n", |
|
|
" <td>1.10330</td>\n", |
|
|
" <td>1.091670</td>\n", |
|
|
" <td>1.103275</td>\n", |
|
|
" <td>1.09166</td>\n", |
|
|
" <td>1.103340</td>\n", |
|
|
" <td>1.09176</td>\n", |
|
|
" <td>61</td>\n", |
|
|
" <td>84</td>\n", |
|
|
" </tr>\n", |
|
|
" </tr>\n", |
|
|
" <tr>\n", |
|
|
" <tr>\n", |
|
|
" <th>2999996</th>\n", |
|
|
" <th>999996</th>\n", |
|
|
" <td>8230799</td>\n", |
|
|
" <td>7857775</td>\n", |
|
|
" <td>2023-05-03 10:35:15</td>\n", |
|
|
" <td>2023-02-01 17:06:15</td>\n", |
|
|
" <td>1683110130000000000</td>\n", |
|
|
" <td>1675271190000000000</td>\n", |
|
|
" <td>2023-05-03 10:35:30</td>\n", |
|
|
" <td>2023-02-01 17:06:30</td>\n", |
|
|
" <td>1.103300</td>\n", |
|
|
" <td>1.091680</td>\n", |
|
|
" <td>1.10341</td>\n", |
|
|
" <td>1.091660</td>\n", |
|
|
" <td>1.103300</td>\n", |
|
|
" <td>1.09165</td>\n", |
|
|
" <td>1.103410</td>\n", |
|
|
" <td>1.09168</td>\n", |
|
|
" <td>44</td>\n", |
|
|
" <td>51</td>\n", |
|
|
" </tr>\n", |
|
|
" </tr>\n", |
|
|
" <tr>\n", |
|
|
" <tr>\n", |
|
|
" <th>2999997</th>\n", |
|
|
" <th>999997</th>\n", |
|
|
" <td>8230799</td>\n", |
|
|
" <td>7857775</td>\n", |
|
|
" <td>2023-05-03 10:35:15</td>\n", |
|
|
" <td>2023-02-01 17:06:15</td>\n", |
|
|
" <td>1683110130000000000</td>\n", |
|
|
" <td>1675271190000000000</td>\n", |
|
|
" <td>2023-05-03 10:35:30</td>\n", |
|
|
" <td>2023-02-01 17:06:30</td>\n", |
|
|
" <td>1.103300</td>\n", |
|
|
" <td>1.091680</td>\n", |
|
|
" <td>1.10341</td>\n", |
|
|
" <td>1.091660</td>\n", |
|
|
" <td>1.103300</td>\n", |
|
|
" <td>1.09165</td>\n", |
|
|
" <td>1.103410</td>\n", |
|
|
" <td>1.09168</td>\n", |
|
|
" <td>44</td>\n", |
|
|
" <td>51</td>\n", |
|
|
" </tr>\n", |
|
|
" </tr>\n", |
|
|
" <tr>\n", |
|
|
" <tr>\n", |
|
|
" <th>2999998</th>\n", |
|
|
" <th>999998</th>\n", |
|
|
" <td>8230800</td>\n", |
|
|
" <td>7857776</td>\n", |
|
|
" <td>2023-05-03 10:35:30</td>\n", |
|
|
" <td>2023-02-01 17:06:30</td>\n", |
|
|
" <td>1683110145000000000</td>\n", |
|
|
" <td>1675271205000000000</td>\n", |
|
|
" <td>2023-05-03 10:35:45</td>\n", |
|
|
" <td>2023-02-01 17:06:45</td>\n", |
|
|
" <td>1.103415</td>\n", |
|
|
" <td>1.091660</td>\n", |
|
|
" <td>1.10351</td>\n", |
|
|
" <td>1.091655</td>\n", |
|
|
" <td>1.103385</td>\n", |
|
|
" <td>1.09164</td>\n", |
|
|
" <td>1.103515</td>\n", |
|
|
" <td>1.09168</td>\n", |
|
|
" <td>51</td>\n", |
|
|
" <td>63</td>\n", |
|
|
" </tr>\n", |
|
|
" </tr>\n", |
|
|
" <tr>\n", |
|
|
" <tr>\n", |
|
|
" <th>2999999</th>\n", |
|
|
" <th>999999</th>\n", |
|
|
" <td>8230800</td>\n", |
|
|
" <td>7857776</td>\n", |
|
|
" <td>2023-05-03 10:35:30</td>\n", |
|
|
" <td>2023-02-01 17:06:30</td>\n", |
|
|
" <td>1683110145000000000</td>\n", |
|
|
" <td>1675271205000000000</td>\n", |
|
|
" <td>2023-05-03 10:35:45</td>\n", |
|
|
" <td>2023-02-01 17:06:45</td>\n", |
|
|
" <td>1.103415</td>\n", |
|
|
" <td>1.091660</td>\n", |
|
|
" <td>1.10351</td>\n", |
|
|
" <td>1.091655</td>\n", |
|
|
" <td>1.103385</td>\n", |
|
|
" <td>1.09164</td>\n", |
|
|
" <td>1.103515</td>\n", |
|
|
" <td>1.09168</td>\n", |
|
|
" <td>51</td>\n", |
|
|
" <td>63</td>\n", |
|
|
" </tr>\n", |
|
|
" </tr>\n", |
|
|
" </tbody>\n", |
|
|
" </tbody>\n", |
|
|
"</table>\n", |
|
|
"</table>\n", |
|
|
@ -512,21 +514,21 @@ |
|
|
], |
|
|
], |
|
|
"text/plain": [ |
|
|
"text/plain": [ |
|
|
" id from at to \\\n", |
|
|
" id from at to \\\n", |
|
|
"2999995 8230798 2023-05-03 10:35:00 1683110115000000000 2023-05-03 10:35:15 \n", |
|
|
"999995 7857774 2023-02-01 17:06:00 1675271175000000000 2023-02-01 17:06:15 \n", |
|
|
"2999996 8230799 2023-05-03 10:35:15 1683110130000000000 2023-05-03 10:35:30 \n", |
|
|
"999996 7857775 2023-02-01 17:06:15 1675271190000000000 2023-02-01 17:06:30 \n", |
|
|
"2999997 8230799 2023-05-03 10:35:15 1683110130000000000 2023-05-03 10:35:30 \n", |
|
|
"999997 7857775 2023-02-01 17:06:15 1675271190000000000 2023-02-01 17:06:30 \n", |
|
|
"2999998 8230800 2023-05-03 10:35:30 1683110145000000000 2023-05-03 10:35:45 \n", |
|
|
"999998 7857776 2023-02-01 17:06:30 1675271205000000000 2023-02-01 17:06:45 \n", |
|
|
"2999999 8230800 2023-05-03 10:35:30 1683110145000000000 2023-05-03 10:35:45 \n", |
|
|
"999999 7857776 2023-02-01 17:06:30 1675271205000000000 2023-02-01 17:06:45 \n", |
|
|
"\n", |
|
|
"\n", |
|
|
" open close min max volume \n", |
|
|
" open close min max volume \n", |
|
|
"2999995 1.103340 1.10330 1.103275 1.103340 61 \n", |
|
|
"999995 1.091725 1.091670 1.09166 1.09176 84 \n", |
|
|
"2999996 1.103300 1.10341 1.103300 1.103410 44 \n", |
|
|
"999996 1.091680 1.091660 1.09165 1.09168 51 \n", |
|
|
"2999997 1.103300 1.10341 1.103300 1.103410 44 \n", |
|
|
"999997 1.091680 1.091660 1.09165 1.09168 51 \n", |
|
|
"2999998 1.103415 1.10351 1.103385 1.103515 51 \n", |
|
|
"999998 1.091660 1.091655 1.09164 1.09168 63 \n", |
|
|
"2999999 1.103415 1.10351 1.103385 1.103515 51 " |
|
|
"999999 1.091660 1.091655 1.09164 1.09168 63 " |
|
|
] |
|
|
] |
|
|
}, |
|
|
}, |
|
|
"execution_count": 23, |
|
|
"execution_count": 11, |
|
|
"metadata": {}, |
|
|
"metadata": {}, |
|
|
"output_type": "execute_result" |
|
|
"output_type": "execute_result" |
|
|
} |
|
|
} |
|
|
@ -537,7 +539,7 @@ |
|
|
}, |
|
|
}, |
|
|
{ |
|
|
{ |
|
|
"cell_type": "code", |
|
|
"cell_type": "code", |
|
|
"execution_count": 26, |
|
|
"execution_count": 12, |
|
|
"id": "86794e47-611f-4ca8-a7e8-07e71afafe67", |
|
|
"id": "86794e47-611f-4ca8-a7e8-07e71afafe67", |
|
|
"metadata": { |
|
|
"metadata": { |
|
|
"tags": [] |
|
|
"tags": [] |
|
|
@ -547,7 +549,7 @@ |
|
|
"name": "stdout", |
|
|
"name": "stdout", |
|
|
"output_type": "stream", |
|
|
"output_type": "stream", |
|
|
"text": [ |
|
|
"text": [ |
|
|
"10.64297915700081\n" |
|
|
"5.7032926019999195\n" |
|
|
] |
|
|
] |
|
|
} |
|
|
} |
|
|
], |
|
|
], |
|
|
@ -557,7 +559,7 @@ |
|
|
}, |
|
|
}, |
|
|
{ |
|
|
{ |
|
|
"cell_type": "code", |
|
|
"cell_type": "code", |
|
|
"execution_count": 27, |
|
|
"execution_count": 13, |
|
|
"id": "e7926062-8e84-4d3f-90a9-32807ce4f3d4", |
|
|
"id": "e7926062-8e84-4d3f-90a9-32807ce4f3d4", |
|
|
"metadata": { |
|
|
"metadata": { |
|
|
"tags": [] |
|
|
"tags": [] |
|
|
@ -567,7 +569,7 @@ |
|
|
"name": "stdout", |
|
|
"name": "stdout", |
|
|
"output_type": "stream", |
|
|
"output_type": "stream", |
|
|
"text": [ |
|
|
"text": [ |
|
|
"6.190685558998666\n" |
|
|
"6.242225094000105\n" |
|
|
] |
|
|
] |
|
|
} |
|
|
} |
|
|
], |
|
|
], |
|
|
@ -593,15 +595,14 @@ |
|
|
} |
|
|
} |
|
|
], |
|
|
], |
|
|
"source": [ |
|
|
"source": [ |
|
|
"%%time\n", |
|
|
"# %%time\n", |
|
|
"dfCh = cHouseQueryDf(dbname)" |
|
|
"# dfCh = cHouseQueryDf(dbname)" |
|
|
] |
|
|
] |
|
|
}, |
|
|
}, |
|
|
{ |
|
|
{ |
|
|
"cell_type": "markdown", |
|
|
"cell_type": "markdown", |
|
|
"id": "1d389546-911f-43f7-aad1-49f7bcc83503", |
|
|
"id": "1d389546-911f-43f7-aad1-49f7bcc83503", |
|
|
"metadata": { |
|
|
"metadata": { |
|
|
"jp-MarkdownHeadingCollapsed": true, |
|
|
|
|
|
"tags": [] |
|
|
"tags": [] |
|
|
}, |
|
|
}, |
|
|
"source": [ |
|
|
"source": [ |
|
|
@ -610,61 +611,325 @@ |
|
|
}, |
|
|
}, |
|
|
{ |
|
|
{ |
|
|
"cell_type": "code", |
|
|
"cell_type": "code", |
|
|
"execution_count": null, |
|
|
"execution_count": 54, |
|
|
"id": "c3e7ebfd-76f1-4ac4-9833-312eb1a531af", |
|
|
"id": "c3e7ebfd-76f1-4ac4-9833-312eb1a531af", |
|
|
"metadata": {}, |
|
|
"metadata": {}, |
|
|
"outputs": [], |
|
|
"outputs": [], |
|
|
"source": [ |
|
|
"source": [ |
|
|
"client = influxdb_client.InfluxDBClient(\n", |
|
|
"def influxdbConnect():\n", |
|
|
|
|
|
" client = influxdb_client.InfluxDBClient(\n", |
|
|
" url=InfluxDBUrl, token=InfluxDBKey, org=InfluxDBUser\n", |
|
|
" url=InfluxDBUrl, token=InfluxDBKey, org=InfluxDBUser\n", |
|
|
")" |
|
|
" )\n", |
|
|
|
|
|
" return client\n", |
|
|
|
|
|
"\n", |
|
|
|
|
|
"\n", |
|
|
|
|
|
"def influxdbLoadCsv(csv=\"out.csv\", dictDates=[\"from\", \"to\"], index=\"from\"):\n", |
|
|
|
|
|
" # Read data from CSV without index and parse 'TimeStamp' as date.\n", |
|
|
|
|
|
" df = pd.read_csv(csv, sep=\",\", index_col=False, parse_dates=dictDates)\n", |
|
|
|
|
|
" # Set 'TimeStamp' field as index of dataframe # test another indexs\n", |
|
|
|
|
|
" df.set_index(index, inplace=True)\n", |
|
|
|
|
|
" return df\n", |
|
|
|
|
|
"\n", |
|
|
|
|
|
"\n", |
|
|
|
|
|
"def influxdbWriteCsv(dataFrame, bucket, measurement=\"id\", tag=\"volume\"):\n", |
|
|
|
|
|
" client = influxdbConnect()\n", |
|
|
|
|
|
" # write_options=SYNCHRONOUS\n", |
|
|
|
|
|
" with client.write_api() as writer:\n", |
|
|
|
|
|
" writer.write(\n", |
|
|
|
|
|
" bucket=bucket,\n", |
|
|
|
|
|
" record=dataFrame,\n", |
|
|
|
|
|
" data_frame_measurement_name=measurement,\n", |
|
|
|
|
|
" data_frame_tag_columns=[tag],\n", |
|
|
|
|
|
" )\n", |
|
|
|
|
|
" writer.__del__()\n", |
|
|
|
|
|
" client.__del__()\n", |
|
|
|
|
|
" return 0" |
|
|
] |
|
|
] |
|
|
}, |
|
|
}, |
|
|
{ |
|
|
{ |
|
|
"cell_type": "code", |
|
|
"cell_type": "code", |
|
|
"execution_count": null, |
|
|
"execution_count": 55, |
|
|
"id": "cbf61f12-830b-4c57-804a-2257d8b3599a", |
|
|
"id": "e05266b8-ff32-462c-b059-325a40a53d25", |
|
|
|
|
|
"metadata": { |
|
|
|
|
|
"tags": [] |
|
|
|
|
|
}, |
|
|
|
|
|
"outputs": [], |
|
|
|
|
|
"source": [ |
|
|
|
|
|
"client = influxdbLoadCsv()" |
|
|
|
|
|
] |
|
|
|
|
|
}, |
|
|
|
|
|
{ |
|
|
|
|
|
"cell_type": "code", |
|
|
|
|
|
"execution_count": 51, |
|
|
|
|
|
"id": "71300e86-2e39-4c71-8b0c-6d6e21728e48", |
|
|
"metadata": { |
|
|
"metadata": { |
|
|
"tags": [] |
|
|
"tags": [] |
|
|
}, |
|
|
}, |
|
|
"outputs": [], |
|
|
"outputs": [], |
|
|
"source": [ |
|
|
"source": [ |
|
|
"# Read data from CSV without index and parse 'TimeStamp' as date.\n", |
|
|
"# write from db and benchmark time\n", |
|
|
"df = pd.read_csv(\"out.csv\", sep=\",\", index_col=False, parse_dates=[\"from\"])\n", |
|
|
"start = timeit.default_timer()\n", |
|
|
"# Set 'TimeStamp' field as index of dataframe # test another indexs\n", |
|
|
"influxdbWriteCsv(df, InfluxDBBucket)\n", |
|
|
"df.set_index(\"from\", inplace=True)" |
|
|
"# dfIdw = cHouseQueryDf(dbname)\n", |
|
|
|
|
|
"stop = timeit.default_timer()\n", |
|
|
|
|
|
"influxdb_write_execution_time = stop - start" |
|
|
|
|
|
] |
|
|
|
|
|
}, |
|
|
|
|
|
{ |
|
|
|
|
|
"cell_type": "code", |
|
|
|
|
|
"execution_count": 52, |
|
|
|
|
|
"id": "c8515d8e-2f34-4a2f-b9be-3faefef2d0ca", |
|
|
|
|
|
"metadata": { |
|
|
|
|
|
"tags": [] |
|
|
|
|
|
}, |
|
|
|
|
|
"outputs": [ |
|
|
|
|
|
{ |
|
|
|
|
|
"name": "stdout", |
|
|
|
|
|
"output_type": "stream", |
|
|
|
|
|
"text": [ |
|
|
|
|
|
"89.17334011999992\n" |
|
|
|
|
|
] |
|
|
|
|
|
} |
|
|
|
|
|
], |
|
|
|
|
|
"source": [ |
|
|
|
|
|
"print(influxdb_write_execution_time)" |
|
|
|
|
|
] |
|
|
|
|
|
}, |
|
|
|
|
|
{ |
|
|
|
|
|
"cell_type": "code", |
|
|
|
|
|
"execution_count": 60, |
|
|
|
|
|
"id": "ecd217ab-0e16-40a6-9b92-9212b9bb20e9", |
|
|
|
|
|
"metadata": { |
|
|
|
|
|
"tags": [] |
|
|
|
|
|
}, |
|
|
|
|
|
"outputs": [], |
|
|
|
|
|
"source": [ |
|
|
|
|
|
"query = \"\"\"\n", |
|
|
|
|
|
"from(bucket: \"system\")\n", |
|
|
|
|
|
"|> range(start:2023-03-03T18:14:30Z, stop: now())\n", |
|
|
|
|
|
"|> filter(fn: (r) => r._measurement == \"id\")\n", |
|
|
|
|
|
"|> filter(fn: (r) => r._field == \"volume\")\n", |
|
|
|
|
|
"|> pivot(rowKey:[\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\")\"\"\"\n", |
|
|
|
|
|
"# |> filter(fn: (r) => r._field == \"volume\")\n", |
|
|
|
|
|
"# |> filter(fn: (r) => r.cpu == \"cpu-total\")" |
|
|
|
|
|
] |
|
|
|
|
|
}, |
|
|
|
|
|
{ |
|
|
|
|
|
"cell_type": "code", |
|
|
|
|
|
"execution_count": 58, |
|
|
|
|
|
"id": "09028588-e450-491c-8b84-246807dad9c3", |
|
|
|
|
|
"metadata": { |
|
|
|
|
|
"tags": [] |
|
|
|
|
|
}, |
|
|
|
|
|
"outputs": [], |
|
|
|
|
|
"source": [ |
|
|
|
|
|
"# 2021-01-01T00:00:00Z\n", |
|
|
|
|
|
"# 2023-03-03T18:14:30Z\n", |
|
|
|
|
|
"\n", |
|
|
|
|
|
"\n", |
|
|
|
|
|
"def influxdRead(org=InfluxDBUrl, query=query):\n", |
|
|
|
|
|
" client = influxdbConnect()\n", |
|
|
|
|
|
" InfluxDf = client.query_api().query_data_frame(org=org, query=query)\n", |
|
|
|
|
|
" display(InfluxDf.head())\n", |
|
|
|
|
|
" return InfluxDf" |
|
|
] |
|
|
] |
|
|
}, |
|
|
}, |
|
|
{ |
|
|
{ |
|
|
"cell_type": "code", |
|
|
"cell_type": "code", |
|
|
"execution_count": null, |
|
|
"execution_count": null, |
|
|
"id": "54342a28-ba2b-4ade-a692-00566b53a639", |
|
|
"id": "8f28e32f-59d3-427b-8fe9-3d8bdf0ab675", |
|
|
"metadata": { |
|
|
"metadata": { |
|
|
"tags": [] |
|
|
"tags": [] |
|
|
}, |
|
|
}, |
|
|
"outputs": [], |
|
|
"outputs": [], |
|
|
"source": [ |
|
|
"source": [ |
|
|
"df.head()" |
|
|
"dfIdb = influxdRead()\n", |
|
|
|
|
|
"dfIdb.head()" |
|
|
] |
|
|
] |
|
|
}, |
|
|
}, |
|
|
{ |
|
|
{ |
|
|
"cell_type": "code", |
|
|
"cell_type": "code", |
|
|
"execution_count": null, |
|
|
"execution_count": null, |
|
|
"id": "f861fab2-f1b1-49dd-b758-12d10aef3462", |
|
|
"id": "86a950d2-1c8e-46d5-aafe-444beb88bb88", |
|
|
"metadata": {}, |
|
|
"metadata": {}, |
|
|
"outputs": [], |
|
|
"outputs": [], |
|
|
"source": [ |
|
|
"source": [ |
|
|
"%%time\n", |
|
|
"# read from db and benchmark time\n", |
|
|
"# gravando... demorou... mas deu certo\n", |
|
|
"start = timeit.default_timer()\n", |
|
|
"with client.write_api() as writer:\n", |
|
|
"dfIdr = cHouseQueryDf(dbname)\n", |
|
|
" writer.write(\n", |
|
|
"stop = timeit.default_timer()\n", |
|
|
" bucket=InfluxDBBucket,\n", |
|
|
"influxdb_read_execution_time = stop - start" |
|
|
" record=df,\n", |
|
|
] |
|
|
" data_frame_measurement_name=\"id\",\n", |
|
|
}, |
|
|
" data_frame_tag_columns=[\"volume\"],\n", |
|
|
{ |
|
|
" )" |
|
|
"cell_type": "code", |
|
|
|
|
|
"execution_count": 19, |
|
|
|
|
|
"id": "54342a28-ba2b-4ade-a692-00566b53a639", |
|
|
|
|
|
"metadata": { |
|
|
|
|
|
"tags": [] |
|
|
|
|
|
}, |
|
|
|
|
|
"outputs": [ |
|
|
|
|
|
{ |
|
|
|
|
|
"data": { |
|
|
|
|
|
"text/html": [ |
|
|
|
|
|
"<div>\n", |
|
|
|
|
|
"<style scoped>\n", |
|
|
|
|
|
" .dataframe tbody tr th:only-of-type {\n", |
|
|
|
|
|
" vertical-align: middle;\n", |
|
|
|
|
|
" }\n", |
|
|
|
|
|
"\n", |
|
|
|
|
|
" .dataframe tbody tr th {\n", |
|
|
|
|
|
" vertical-align: top;\n", |
|
|
|
|
|
" }\n", |
|
|
|
|
|
"\n", |
|
|
|
|
|
" .dataframe thead th {\n", |
|
|
|
|
|
" text-align: right;\n", |
|
|
|
|
|
" }\n", |
|
|
|
|
|
"</style>\n", |
|
|
|
|
|
"<table border=\"1\" class=\"dataframe\">\n", |
|
|
|
|
|
" <thead>\n", |
|
|
|
|
|
" <tr style=\"text-align: right;\">\n", |
|
|
|
|
|
" <th></th>\n", |
|
|
|
|
|
" <th>Unnamed: 0</th>\n", |
|
|
|
|
|
" <th>id</th>\n", |
|
|
|
|
|
" <th>at</th>\n", |
|
|
|
|
|
" <th>to</th>\n", |
|
|
|
|
|
" <th>open</th>\n", |
|
|
|
|
|
" <th>close</th>\n", |
|
|
|
|
|
" <th>min</th>\n", |
|
|
|
|
|
" <th>max</th>\n", |
|
|
|
|
|
" <th>volume</th>\n", |
|
|
|
|
|
" </tr>\n", |
|
|
|
|
|
" <tr>\n", |
|
|
|
|
|
" <th>from</th>\n", |
|
|
|
|
|
" <th></th>\n", |
|
|
|
|
|
" <th></th>\n", |
|
|
|
|
|
" <th></th>\n", |
|
|
|
|
|
" <th></th>\n", |
|
|
|
|
|
" <th></th>\n", |
|
|
|
|
|
" <th></th>\n", |
|
|
|
|
|
" <th></th>\n", |
|
|
|
|
|
" <th></th>\n", |
|
|
|
|
|
" <th></th>\n", |
|
|
|
|
|
" </tr>\n", |
|
|
|
|
|
" </thead>\n", |
|
|
|
|
|
" <tbody>\n", |
|
|
|
|
|
" <tr>\n", |
|
|
|
|
|
" <th>2023-03-03 18:13:30</th>\n", |
|
|
|
|
|
" <td>999995</td>\n", |
|
|
|
|
|
" <td>7984748</td>\n", |
|
|
|
|
|
" <td>1677867225000000000</td>\n", |
|
|
|
|
|
" <td>2023-03-03 18:13:45</td>\n", |
|
|
|
|
|
" <td>1.062695</td>\n", |
|
|
|
|
|
" <td>1.062635</td>\n", |
|
|
|
|
|
" <td>1.062630</td>\n", |
|
|
|
|
|
" <td>1.062700</td>\n", |
|
|
|
|
|
" <td>64</td>\n", |
|
|
|
|
|
" </tr>\n", |
|
|
|
|
|
" <tr>\n", |
|
|
|
|
|
" <th>2023-03-03 18:13:45</th>\n", |
|
|
|
|
|
" <td>999996</td>\n", |
|
|
|
|
|
" <td>7984749</td>\n", |
|
|
|
|
|
" <td>1677867240000000000</td>\n", |
|
|
|
|
|
" <td>2023-03-03 18:14:00</td>\n", |
|
|
|
|
|
" <td>1.062645</td>\n", |
|
|
|
|
|
" <td>1.062650</td>\n", |
|
|
|
|
|
" <td>1.062625</td>\n", |
|
|
|
|
|
" <td>1.062650</td>\n", |
|
|
|
|
|
" <td>43</td>\n", |
|
|
|
|
|
" </tr>\n", |
|
|
|
|
|
" <tr>\n", |
|
|
|
|
|
" <th>2023-03-03 18:14:00</th>\n", |
|
|
|
|
|
" <td>999997</td>\n", |
|
|
|
|
|
" <td>7984750</td>\n", |
|
|
|
|
|
" <td>1677867255000000000</td>\n", |
|
|
|
|
|
" <td>2023-03-03 18:14:15</td>\n", |
|
|
|
|
|
" <td>1.062640</td>\n", |
|
|
|
|
|
" <td>1.062625</td>\n", |
|
|
|
|
|
" <td>1.062620</td>\n", |
|
|
|
|
|
" <td>1.062665</td>\n", |
|
|
|
|
|
" <td>47</td>\n", |
|
|
|
|
|
" </tr>\n", |
|
|
|
|
|
" <tr>\n", |
|
|
|
|
|
" <th>2023-03-03 18:14:15</th>\n", |
|
|
|
|
|
" <td>999998</td>\n", |
|
|
|
|
|
" <td>7984751</td>\n", |
|
|
|
|
|
" <td>1677867270000000000</td>\n", |
|
|
|
|
|
" <td>2023-03-03 18:14:30</td>\n", |
|
|
|
|
|
" <td>1.062625</td>\n", |
|
|
|
|
|
" <td>1.062535</td>\n", |
|
|
|
|
|
" <td>1.062535</td>\n", |
|
|
|
|
|
" <td>1.062645</td>\n", |
|
|
|
|
|
" <td>43</td>\n", |
|
|
|
|
|
" </tr>\n", |
|
|
|
|
|
" <tr>\n", |
|
|
|
|
|
" <th>2023-03-03 18:14:30</th>\n", |
|
|
|
|
|
" <td>999999</td>\n", |
|
|
|
|
|
" <td>7984752</td>\n", |
|
|
|
|
|
" <td>1677867285000000000</td>\n", |
|
|
|
|
|
" <td>2023-03-03 18:14:45</td>\n", |
|
|
|
|
|
" <td>1.062535</td>\n", |
|
|
|
|
|
" <td>1.062520</td>\n", |
|
|
|
|
|
" <td>1.062520</td>\n", |
|
|
|
|
|
" <td>1.062580</td>\n", |
|
|
|
|
|
" <td>59</td>\n", |
|
|
|
|
|
" </tr>\n", |
|
|
|
|
|
" </tbody>\n", |
|
|
|
|
|
"</table>\n", |
|
|
|
|
|
"</div>" |
|
|
|
|
|
], |
|
|
|
|
|
"text/plain": [ |
|
|
|
|
|
" Unnamed: 0 id at \\\n", |
|
|
|
|
|
"from \n", |
|
|
|
|
|
"2023-03-03 18:13:30 999995 7984748 1677867225000000000 \n", |
|
|
|
|
|
"2023-03-03 18:13:45 999996 7984749 1677867240000000000 \n", |
|
|
|
|
|
"2023-03-03 18:14:00 999997 7984750 1677867255000000000 \n", |
|
|
|
|
|
"2023-03-03 18:14:15 999998 7984751 1677867270000000000 \n", |
|
|
|
|
|
"2023-03-03 18:14:30 999999 7984752 1677867285000000000 \n", |
|
|
|
|
|
"\n", |
|
|
|
|
|
" to open close min \\\n", |
|
|
|
|
|
"from \n", |
|
|
|
|
|
"2023-03-03 18:13:30 2023-03-03 18:13:45 1.062695 1.062635 1.062630 \n", |
|
|
|
|
|
"2023-03-03 18:13:45 2023-03-03 18:14:00 1.062645 1.062650 1.062625 \n", |
|
|
|
|
|
"2023-03-03 18:14:00 2023-03-03 18:14:15 1.062640 1.062625 1.062620 \n", |
|
|
|
|
|
"2023-03-03 18:14:15 2023-03-03 18:14:30 1.062625 1.062535 1.062535 \n", |
|
|
|
|
|
"2023-03-03 18:14:30 2023-03-03 18:14:45 1.062535 1.062520 1.062520 \n", |
|
|
|
|
|
"\n", |
|
|
|
|
|
" max volume \n", |
|
|
|
|
|
"from \n", |
|
|
|
|
|
"2023-03-03 18:13:30 1.062700 64 \n", |
|
|
|
|
|
"2023-03-03 18:13:45 1.062650 43 \n", |
|
|
|
|
|
"2023-03-03 18:14:00 1.062665 47 \n", |
|
|
|
|
|
"2023-03-03 18:14:15 1.062645 43 \n", |
|
|
|
|
|
"2023-03-03 18:14:30 1.062580 59 " |
|
|
|
|
|
] |
|
|
|
|
|
}, |
|
|
|
|
|
"execution_count": 19, |
|
|
|
|
|
"metadata": {}, |
|
|
|
|
|
"output_type": "execute_result" |
|
|
|
|
|
} |
|
|
|
|
|
], |
|
|
|
|
|
"source": [ |
|
|
|
|
|
"df.tail()" |
|
|
] |
|
|
] |
|
|
}, |
|
|
}, |
|
|
|
|
|
{ |
|
|
|
|
|
"cell_type": "code", |
|
|
|
|
|
"execution_count": 20, |
|
|
|
|
|
"id": "f861fab2-f1b1-49dd-b758-12d10aef3462", |
|
|
|
|
|
"metadata": {}, |
|
|
|
|
|
"outputs": [ |
|
|
|
|
|
{ |
|
|
|
|
|
"name": "stdout", |
|
|
|
|
|
"output_type": "stream", |
|
|
|
|
|
"text": [ |
|
|
|
|
|
"CPU times: user 25.8 s, sys: 408 ms, total: 26.2 s\n", |
|
|
|
|
|
"Wall time: 1min 36s\n" |
|
|
|
|
|
] |
|
|
|
|
|
} |
|
|
|
|
|
], |
|
|
|
|
|
"source": [] |
|
|
|
|
|
}, |
|
|
{ |
|
|
{ |
|
|
"cell_type": "code", |
|
|
"cell_type": "code", |
|
|
"execution_count": null, |
|
|
"execution_count": null, |
|
|
|