datapoint/compare-timeseries-database
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

finish dusckdb Functions code

Browse Source
This commit is contained in:
flashlan
2023-06-21 23:28:22 -03:00
parent 0f0ee40e70
commit 86404adc46
1 changed files with 70 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files
compareDBs.ipynb
+70 -64
View File
@@ -1368,7 +1368,6 @@
"cell_type": "markdown", "cell_type": "markdown",
"id": "50d1fc58-89a7-4507-aff0-6e943656cfe0", "id": "50d1fc58-89a7-4507-aff0-6e943656cfe0",
"metadata": { "metadata": {
"jp-MarkdownHeadingCollapsed": true,
"tags": [] "tags": []
}, },
"source": [ "source": [
@@ -1515,7 +1514,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 61, "execution_count": 85,
"id": "bbcdb883-d6dc-46db-88db-4c90b84522ba", "id": "bbcdb883-d6dc-46db-88db-4c90b84522ba",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
@@ -1542,17 +1541,17 @@
" return 0\n", " return 0\n",
"\n", "\n",
"\n", "\n",
"def duckdbRead():\n", "def duckdbRead(dbfile):\n",
" conn = duckdbConnect()\n", " conn = duckdbConnect()\n",
" conn = duckdb.connect(file)\n", " conn = duckdb.connect(dbfile)\n",
" conn.execute(\"SELECT * FROM EURUSD\").fetchdf()\n", " conn.execute(\"SELECT * FROM {}\".format(dbname)).fetchdf()\n",
" conn.close()\n", " conn.close()\n",
" return 0" " return 0"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 60, "execution_count": 73,
"id": "1c787f48-5640-4eb5-9456-be8f0a8211eb", "id": "1c787f48-5640-4eb5-9456-be8f0a8211eb",
"metadata": { "metadata": {
"tags": [] "tags": []
@@ -1564,32 +1563,43 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 68, "execution_count": null,
"id": "f07f03d0-021e-4dc3-bfa8-efc029a9797a", "id": "f07f03d0-021e-4dc3-bfa8-efc029a9797a",
"metadata": { "metadata": {
"tags": [] "tags": []
}, },
"outputs": [],
"source": [
"conn = duckdbConnect()\n",
"conn.execute(\"DROP TABLE EURUSD\")"
]
},
{
"cell_type": "code",
"execution_count": 74,
"id": "45620be8-34c6-4a3a-89f7-09337f665c90",
"metadata": {
"tags": []
},
"outputs": [ "outputs": [
{ {
"ename": "CatalogException", "data": {
"evalue": "Catalog Error: Table with name EURUSDtest does not exist!\nDid you mean \"temp.information_schema.tables\"?", "text/plain": [
"output_type": "error", "<duckdb.DuckDBPyConnection at 0x7f925fb29b30>"
"traceback": [ ]
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", },
"\u001b[0;31mCatalogException\u001b[0m Traceback (most recent call last)", "execution_count": 74,
"Cell \u001b[0;32mIn[68], line 2\u001b[0m\n\u001b[1;32m 1\u001b[0m conn \u001b[38;5;241m=\u001b[39m duckdbConnect()\n\u001b[0;32m----> 2\u001b[0m \u001b[43mconn\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexecute\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mDROP TABLE EURUSDtest\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n", "metadata": {},
"\u001b[0;31mCatalogException\u001b[0m: Catalog Error: Table with name EURUSDtest does not exist!\nDid you mean \"temp.information_schema.tables\"?" "output_type": "execute_result"
]
} }
], ],
"source": [ "source": [
"conn = duckdbConnect()\n", "conn.register(\"EURUSDtest\", data)"
"conn.execute(\"DROP TABLE EURUSDtest\")"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 71, "execution_count": 84,
"id": "9eb19431-fbad-43b4-84f7-0043e65de162", "id": "9eb19431-fbad-43b4-84f7-0043e65de162",
"metadata": { "metadata": {
"tags": [] "tags": []
@@ -1622,25 +1632,15 @@
" <tbody>\n", " <tbody>\n",
" <tr>\n", " <tr>\n",
" <th>0</th>\n", " <th>0</th>\n",
" <td>EURUSD</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>EURUSDtest</td>\n", " <td>EURUSDtest</td>\n",
" </tr>\n", " </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>test</td>\n",
" </tr>\n",
" </tbody>\n", " </tbody>\n",
"</table>\n", "</table>\n",
"</div>" "</div>"
], ],
"text/plain": [ "text/plain": [
" name\n", " name\n",
"0 EURUSD\n", "0 EURUSDtest"
"1 EURUSDtest\n",
"2 test"
] ]
}, },
"metadata": {}, "metadata": {},
@@ -1649,30 +1649,19 @@
], ],
"source": [ "source": [
"conn = duckdb.connect(\"file.db\")\n", "conn = duckdb.connect(\"file.db\")\n",
"# conn.execute(\"DROP TABLE EURUSDtest\")\n",
"# conn.sql(\"CREATE TABLE EURUSDtest(i INTEGER)\")\n", "# conn.sql(\"CREATE TABLE EURUSDtest(i INTEGER)\")\n",
"display(conn.execute(\"SHOW TABLES\").df())" "display(conn.execute(\"SHOW TABLES\").df())"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 63, "execution_count": null,
"id": "32037939-def0-48e7-9a47-46bcf1a5883c", "id": "32037939-def0-48e7-9a47-46bcf1a5883c",
"metadata": { "metadata": {
"tags": [] "tags": []
}, },
"outputs": [ "outputs": [],
{
"ename": "CatalogException",
"evalue": "Catalog Error: Table with name EURUSDtest does not exist!\nDid you mean \"temp.information_schema.tables\"?\nLINE 1: SELECT * FROM EURUSDtest\n ^",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mCatalogException\u001b[0m Traceback (most recent call last)",
"Cell \u001b[0;32mIn[63], line 4\u001b[0m\n\u001b[1;32m 2\u001b[0m conn \u001b[38;5;241m=\u001b[39m duckdb\u001b[38;5;241m.\u001b[39mconnect(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfile.db\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 3\u001b[0m conn \u001b[38;5;241m=\u001b[39m duckdbConnect()\n\u001b[0;32m----> 4\u001b[0m \u001b[43mconn\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexecute\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mSELECT * FROM EURUSDtest\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m 5\u001b[0m conn\u001b[38;5;241m.\u001b[39mfetchall()\n",
"\u001b[0;31mCatalogException\u001b[0m: Catalog Error: Table with name EURUSDtest does not exist!\nDid you mean \"temp.information_schema.tables\"?\nLINE 1: SELECT * FROM EURUSDtest\n ^"
]
}
],
"source": [ "source": [
"# conn = duckdbConnect()\n", "# conn = duckdbConnect()\n",
"conn.execute(\"SELECT * FROM EURUSDtest\")\n", "conn.execute(\"SELECT * FROM EURUSDtest\")\n",
@@ -1681,25 +1670,12 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 62, "execution_count": 82,
"id": "c6f53d67-684b-4b34-a573-472986ee3e47", "id": "c6f53d67-684b-4b34-a573-472986ee3e47",
"metadata": { "metadata": {
"tags": [] "tags": []
}, },
"outputs": [ "outputs": [],
{
"ename": "CatalogException",
"evalue": "Catalog Error: Table with name \"EURUSDtest\" already exists!",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mCatalogException\u001b[0m Traceback (most recent call last)",
"Cell \u001b[0;32mIn[62], line 2\u001b[0m\n\u001b[1;32m 1\u001b[0m start \u001b[38;5;241m=\u001b[39m timeit\u001b[38;5;241m.\u001b[39mdefault_timer()\n\u001b[0;32m----> 2\u001b[0m dfDuckdb \u001b[38;5;241m=\u001b[39m \u001b[43mduckdbWrite\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mfile.db\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdbname\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 3\u001b[0m stop \u001b[38;5;241m=\u001b[39m timeit\u001b[38;5;241m.\u001b[39mdefault_timer()\n\u001b[1;32m 4\u001b[0m duckdb_write_execution_time \u001b[38;5;241m=\u001b[39m stop \u001b[38;5;241m-\u001b[39m start\n",
"Cell \u001b[0;32mIn[61], line 18\u001b[0m, in \u001b[0;36mduckdbWrite\u001b[0;34m(file, dataframe, tableName)\u001b[0m\n\u001b[1;32m 15\u001b[0m \u001b[38;5;66;03m# conn.execute(\"DROP TABLE EURUSDtest\")\u001b[39;00m\n\u001b[1;32m 16\u001b[0m \u001b[38;5;66;03m# conn.sql('CREATE TABLE EURUSDtest(i INTEGER)')\u001b[39;00m\n\u001b[1;32m 17\u001b[0m conn\u001b[38;5;241m.\u001b[39mregister(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtempTable\u001b[39m\u001b[38;5;124m\"\u001b[39m, dataframe)\n\u001b[0;32m---> 18\u001b[0m \u001b[43mconn\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexecute\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mCREATE TABLE \u001b[39;49m\u001b[38;5;132;43;01m{}\u001b[39;49;00m\u001b[38;5;124;43m AS SELECT * FROM tempTable\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mformat\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtableName\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 19\u001b[0m conn\u001b[38;5;241m.\u001b[39mclose()\n\u001b[1;32m 20\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;241m0\u001b[39m\n",
"\u001b[0;31mCatalogException\u001b[0m: Catalog Error: Table with name \"EURUSDtest\" already exists!"
]
}
],
"source": [ "source": [
"start = timeit.default_timer()\n", "start = timeit.default_timer()\n",
"dfDuckdb = duckdbWrite(\"file.db\", data, dbname)\n", "dfDuckdb = duckdbWrite(\"file.db\", data, dbname)\n",
@@ -1709,7 +1685,27 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 87,
"id": "102f363a-b35d-433c-8752-7acc85c27bdc",
"metadata": {
"tags": []
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"1.7639581979965442\n"
]
}
],
"source": [
"print(duckdb_write_execution_time)"
]
},
{
"cell_type": "code",
"execution_count": 86,
"id": "f630fc1a-0d52-4e3a-9dfe-1ec60d188033", "id": "f630fc1a-0d52-4e3a-9dfe-1ec60d188033",
"metadata": { "metadata": {
"tags": [] "tags": []
@@ -1717,20 +1713,30 @@
"outputs": [], "outputs": [],
"source": [ "source": [
"start = timeit.default_timer()\n", "start = timeit.default_timer()\n",
"dfDuckdb = duckdbRead()\n", "dfDuckdb = duckdbRead(\"file.db\")\n",
"stop = timeit.default_timer()\n", "stop = timeit.default_timer()\n",
"duckdb_read_execution_time = stop - start" "duckdb_read_execution_time = stop - start"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 88,
"id": "c6abdaaa-3ac2-425b-9208-d6cb79afe966", "id": "c6abdaaa-3ac2-425b-9208-d6cb79afe966",
"metadata": { "metadata": {
"tags": [] "tags": []
}, },
"outputs": [], "outputs": [
"source": [] {
"name": "stdout",
"output_type": "stream",
"text": [
"0.5125257010004134\n"
]
}
],
"source": [
"print(duckdb_read_execution_time)"
]
}, },
{ {
"cell_type": "markdown", "cell_type": "markdown",