From 393f8a2644ef1609d3657dda293046bdeb80456e Mon Sep 17 00:00:00 2001 From: flashlan Date: Fri, 16 Jun 2023 23:43:55 -0300 Subject: [PATCH] Influxdb Read DB Code --- compareDBs.ipynb | 140 +++++++++++++++-------------------------------- 1 file changed, 43 insertions(+), 97 deletions(-) diff --git a/compareDBs.ipynb b/compareDBs.ipynb index ecec002..8cb822a 100644 --- a/compareDBs.ipynb +++ b/compareDBs.ipynb @@ -101,7 +101,7 @@ }, { "cell_type": "code", - "execution_count": 42, + "execution_count": 94, "id": "968403e3-2e5e-4834-b969-be4600e2963a", "metadata": { "tags": [] @@ -306,7 +306,6 @@ "cell_type": "markdown", "id": "4a8d5703-9bc9-4d38-83ff-457159304d58", "metadata": { - "jp-MarkdownHeadingCollapsed": true, "tags": [] }, "source": [ @@ -611,7 +610,7 @@ }, { "cell_type": "code", - "execution_count": 54, + "execution_count": 95, "id": "c3e7ebfd-76f1-4ac4-9833-312eb1a531af", "metadata": {}, "outputs": [], @@ -643,33 +642,38 @@ " )\n", " writer.__del__()\n", " client.__del__()\n", - " return 0" + " return 0\n", + "\n", + "\n", + "def influxdRead(org=InfluxDBUrl, query=query):\n", + " client = influxdbConnect()\n", + " InfluxDf = client.query_api().query_data_frame(org=\"librography\", query=query)\n", + " # display(InfluxDf.head())\n", + " return InfluxDf" ] }, { "cell_type": "code", - "execution_count": 55, + "execution_count": 96, "id": "e05266b8-ff32-462c-b059-325a40a53d25", "metadata": { "tags": [] }, "outputs": [], "source": [ - "client = influxdbLoadCsv()" + "dafr = influxdbLoadCsv()\n", + "# dafr.head()" ] }, { "cell_type": "code", - "execution_count": 51, - "id": "71300e86-2e39-4c71-8b0c-6d6e21728e48", - "metadata": { - "tags": [] - }, + "execution_count": null, + "id": "95191283-f11e-456f-8395-36981ab1ac51", + "metadata": {}, "outputs": [], "source": [ - "# write from db and benchmark time\n", "start = timeit.default_timer()\n", - "influxdbWriteCsv(df, InfluxDBBucket)\n", + "influxdbWriteCsv(dafr, InfluxDBBucket)\n", "# dfIdw = cHouseQueryDf(dbname)\n", "stop = timeit.default_timer()\n", "influxdb_write_execution_time = stop - start" @@ -677,17 +681,15 @@ }, { "cell_type": "code", - "execution_count": 52, - "id": "c8515d8e-2f34-4a2f-b9be-3faefef2d0ca", - "metadata": { - "tags": [] - }, + "execution_count": 119, + "id": "3e5bb029-4988-4692-bc5f-9a6f7fcc2159", + "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "89.17334011999992\n" + "161.51344409900048\n" ] } ], @@ -697,7 +699,7 @@ }, { "cell_type": "code", - "execution_count": 60, + "execution_count": 113, "id": "ecd217ab-0e16-40a6-9b92-9212b9bb20e9", "metadata": { "tags": [] @@ -705,10 +707,9 @@ "outputs": [], "source": [ "query = \"\"\"\n", - "from(bucket: \"system\")\n", + "from(bucket: \"EURUSDtest\")\n", "|> range(start:2023-03-03T18:14:30Z, stop: now())\n", "|> filter(fn: (r) => r._measurement == \"id\")\n", - "|> filter(fn: (r) => r._field == \"volume\")\n", "|> pivot(rowKey:[\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\")\"\"\"\n", "# |> filter(fn: (r) => r._field == \"volume\")\n", "# |> filter(fn: (r) => r.cpu == \"cpu-total\")" @@ -716,49 +717,36 @@ }, { "cell_type": "code", - "execution_count": 58, - "id": "09028588-e450-491c-8b84-246807dad9c3", - "metadata": { - "tags": [] - }, + "execution_count": null, + "id": "850c6921-5e1c-417a-bea6-ea18be642008", + "metadata": {}, "outputs": [], "source": [ - "# 2021-01-01T00:00:00Z\n", - "# 2023-03-03T18:14:30Z\n", - "\n", - "\n", - "def influxdRead(org=InfluxDBUrl, query=query):\n", - " client = influxdbConnect()\n", - " InfluxDf = client.query_api().query_data_frame(org=org, query=query)\n", - " display(InfluxDf.head())\n", - " return InfluxDf" + "# read from db and benchmark time\n", + "start = timeit.default_timer()\n", + "dfIdr = cHouseQueryDf(dbname)\n", + "stop = timeit.default_timer()\n", + "influxdb_read_execution_time = stop - start" ] }, { "cell_type": "code", - "execution_count": null, - "id": "8f28e32f-59d3-427b-8fe9-3d8bdf0ab675", + "execution_count": 118, + "id": "3ee3c0dd-cb70-4124-a0fb-db8dd2c134c0", "metadata": { "tags": [] }, - "outputs": [], - "source": [ - "dfIdb = influxdRead()\n", - "dfIdb.head()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "86a950d2-1c8e-46d5-aafe-444beb88bb88", - "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "183.94615754100232\n" + ] + } + ], "source": [ - "# read from db and benchmark time\n", - "start = timeit.default_timer()\n", - "dfIdr = cHouseQueryDf(dbname)\n", - "stop = timeit.default_timer()\n", - "influxdb_read_execution_time = stop - start" + "print(influxdb_read_execution_time)" ] }, { @@ -913,48 +901,6 @@ "df.tail()" ] }, - { - "cell_type": "code", - "execution_count": 20, - "id": "f861fab2-f1b1-49dd-b758-12d10aef3462", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 25.8 s, sys: 408 ms, total: 26.2 s\n", - "Wall time: 1min 36s\n" - ] - } - ], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "0bb2563d-68e2-4ff4-8842-70ac730dc6b1", - "metadata": {}, - "outputs": [], - "source": [ - "# data\n", - "# |> pivot(\n", - "# rowKey:[\"_time\"],\n", - "# columnKey: [\"_field\"],\n", - "# valueColumn: \"_value\"\n", - "# )" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "bb1596f9-4cee-4642-803a-ee61c9dddf64", - "metadata": {}, - "outputs": [], - "source": [ - "# Read" - ] - }, { "cell_type": "markdown", "id": "b9ddfdc6-c899-4f6c-9b4e-8ec6ab6d7e05",