datapoint
/
compare-timeseries-database
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

mongodb and duckdb functions

master
flashlan 3 years ago
parent
393f8a2644
commit
300c5acd0e
1 changed files with 247 additions and 43 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 288
      compareDBs.ipynb

288
compareDBs.ipynb
View File

@ -602,6 +602,7 @@
"cell_type": "markdown", "cell_type": "markdown",
"id": "1d389546-911f-43f7-aad1-49f7bcc83503", "id": "1d389546-911f-43f7-aad1-49f7bcc83503",
"metadata": { "metadata": {
"jp-MarkdownHeadingCollapsed": true,
"tags": [] "tags": []
}, },
"source": [ "source": [
@ -610,7 +611,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 95, "execution_count": 122,
"id": "c3e7ebfd-76f1-4ac4-9833-312eb1a531af", "id": "c3e7ebfd-76f1-4ac4-9833-312eb1a531af",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
@ -710,28 +711,195 @@
"from(bucket: \"EURUSDtest\")\n", "from(bucket: \"EURUSDtest\")\n",
"|> range(start:2023-03-03T18:14:30Z, stop: now())\n", "|> range(start:2023-03-03T18:14:30Z, stop: now())\n",
"|> filter(fn: (r) => r._measurement == \"id\")\n", "|> filter(fn: (r) => r._measurement == \"id\")\n",
"|> pivot(rowKey:[\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\")\"\"\"\n", "|> pivot(rowKey:[\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\")\"\"\""
"# |> filter(fn: (r) => r._field == \"volume\")\n",
"# |> filter(fn: (r) => r.cpu == \"cpu-total\")"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 120,
"id": "850c6921-5e1c-417a-bea6-ea18be642008", "id": "850c6921-5e1c-417a-bea6-ea18be642008",
"metadata": {
"tags": []
},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>result</th>\n",
" <th>table</th>\n",
" <th>_start</th>\n",
" <th>_stop</th>\n",
" <th>_time</th>\n",
" <th>_measurement</th>\n",
" <th>volume</th>\n",
" <th>Unnamed: 0</th>\n",
" <th>at</th>\n",
" <th>close</th>\n",
" <th>id</th>\n",
" <th>max</th>\n",
" <th>min</th>\n",
" <th>open</th>\n",
" <th>to</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>_result</td>\n",
" <td>0</td>\n",
" <td>2023-03-03 18:14:30+00:00</td>\n",
" <td>2023-06-17 02:47:50.721233+00:00</td>\n",
" <td>2023-03-05 22:01:00+00:00</td>\n",
" <td>id</td>\n",
" <td>0</td>\n",
" <td>115589</td>\n",
" <td>1678053675000000000</td>\n",
" <td>1.063425</td>\n",
" <td>7985654</td>\n",
" <td>1.063425</td>\n",
" <td>1.063425</td>\n",
" <td>1.063425</td>\n",
" <td>2023-03-05 22:01:15</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>_result</td>\n",
" <td>0</td>\n",
" <td>2023-03-03 18:14:30+00:00</td>\n",
" <td>2023-06-17 02:47:50.721233+00:00</td>\n",
" <td>2023-03-05 23:58:30+00:00</td>\n",
" <td>id</td>\n",
" <td>0</td>\n",
" <td>116059</td>\n",
" <td>1678060725000000000</td>\n",
" <td>1.062595</td>\n",
" <td>7986124</td>\n",
" <td>1.062595</td>\n",
" <td>1.062595</td>\n",
" <td>1.062595</td>\n",
" <td>2023-03-05 23:58:45</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>_result</td>\n",
" <td>0</td>\n",
" <td>2023-03-03 18:14:30+00:00</td>\n",
" <td>2023-06-17 02:47:50.721233+00:00</td>\n",
" <td>2023-03-06 23:58:30+00:00</td>\n",
" <td>id</td>\n",
" <td>0</td>\n",
" <td>121819</td>\n",
" <td>1678147125000000000</td>\n",
" <td>1.068615</td>\n",
" <td>7991884</td>\n",
" <td>1.068615</td>\n",
" <td>1.068615</td>\n",
" <td>1.068615</td>\n",
" <td>2023-03-06 23:58:45</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>_result</td>\n",
" <td>0</td>\n",
" <td>2023-03-03 18:14:30+00:00</td>\n",
" <td>2023-06-17 02:47:50.721233+00:00</td>\n",
" <td>2023-03-06 23:59:30+00:00</td>\n",
" <td>id</td>\n",
" <td>0</td>\n",
" <td>121823</td>\n",
" <td>1678147185000000000</td>\n",
" <td>1.068605</td>\n",
" <td>7991888</td>\n",
" <td>1.068605</td>\n",
" <td>1.068605</td>\n",
" <td>1.068605</td>\n",
" <td>2023-03-06 23:59:45</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>_result</td>\n",
" <td>0</td>\n",
" <td>2023-03-03 18:14:30+00:00</td>\n",
" <td>2023-06-17 02:47:50.721233+00:00</td>\n",
" <td>2023-03-08 23:59:00+00:00</td>\n",
" <td>id</td>\n",
" <td>0</td>\n",
" <td>182493</td>\n",
" <td>1678319955000000000</td>\n",
" <td>1.054895</td>\n",
" <td>8003406</td>\n",
" <td>1.054895</td>\n",
" <td>1.054895</td>\n",
" <td>1.054895</td>\n",
" <td>2023-03-08 23:59:15</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" result table _start _stop \\\n",
"0 _result 0 2023-03-03 18:14:30+00:00 2023-06-17 02:47:50.721233+00:00 \n",
"1 _result 0 2023-03-03 18:14:30+00:00 2023-06-17 02:47:50.721233+00:00 \n",
"2 _result 0 2023-03-03 18:14:30+00:00 2023-06-17 02:47:50.721233+00:00 \n",
"3 _result 0 2023-03-03 18:14:30+00:00 2023-06-17 02:47:50.721233+00:00 \n",
"4 _result 0 2023-03-03 18:14:30+00:00 2023-06-17 02:47:50.721233+00:00 \n",
"\n",
" _time _measurement volume Unnamed: 0 \\\n",
"0 2023-03-05 22:01:00+00:00 id 0 115589 \n",
"1 2023-03-05 23:58:30+00:00 id 0 116059 \n",
"2 2023-03-06 23:58:30+00:00 id 0 121819 \n",
"3 2023-03-06 23:59:30+00:00 id 0 121823 \n",
"4 2023-03-08 23:59:00+00:00 id 0 182493 \n",
"\n",
" at close id max min open \\\n",
"0 1678053675000000000 1.063425 7985654 1.063425 1.063425 1.063425 \n",
"1 1678060725000000000 1.062595 7986124 1.062595 1.062595 1.062595 \n",
"2 1678147125000000000 1.068615 7991884 1.068615 1.068615 1.068615 \n",
"3 1678147185000000000 1.068605 7991888 1.068605 1.068605 1.068605 \n",
"4 1678319955000000000 1.054895 8003406 1.054895 1.054895 1.054895 \n",
"\n",
" to \n",
"0 2023-03-05 22:01:15 \n",
"1 2023-03-05 23:58:45 \n",
"2 2023-03-06 23:58:45 \n",
"3 2023-03-06 23:59:45 \n",
"4 2023-03-08 23:59:15 "
]
},
"metadata": {}, "metadata": {},
"outputs": [], "output_type": "display_data"
}
],
"source": [ "source": [
"# read from db and benchmark time\n", "# read from db and benchmark time\n",
"start = timeit.default_timer()\n", "start = timeit.default_timer()\n",
"dfIdr = cHouseQueryDf(dbname)\n", "influxdRead()\n",
"stop = timeit.default_timer()\n", "stop = timeit.default_timer()\n",
"influxdb_read_execution_time = stop - start" "influxdb_read_execution_time = stop - start"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 118, "execution_count": 121,
"id": "3ee3c0dd-cb70-4124-a0fb-db8dd2c134c0", "id": "3ee3c0dd-cb70-4124-a0fb-db8dd2c134c0",
"metadata": { "metadata": {
"tags": [] "tags": []
@ -741,7 +909,7 @@
"name": "stdout", "name": "stdout",
"output_type": "stream", "output_type": "stream",
"text": [ "text": [
"183.94615754100232\n" "181.9998163249984\n"
] ]
} }
], ],
@ -905,6 +1073,7 @@
"cell_type": "markdown", "cell_type": "markdown",
"id": "b9ddfdc6-c899-4f6c-9b4e-8ec6ab6d7e05", "id": "b9ddfdc6-c899-4f6c-9b4e-8ec6ab6d7e05",
"metadata": { "metadata": {
"jp-MarkdownHeadingCollapsed": true,
"tags": [] "tags": []
}, },
"source": [ "source": [
@ -1047,6 +1216,7 @@
"cell_type": "markdown", "cell_type": "markdown",
"id": "f9e0393d-7d1d-406a-a068-9dbf4968e977", "id": "f9e0393d-7d1d-406a-a068-9dbf4968e977",
"metadata": { "metadata": {
"jp-MarkdownHeadingCollapsed": true,
"tags": [] "tags": []
}, },
"source": [ "source": [
@ -1322,63 +1492,84 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 123,
"id": "d104d9af-fa34-4261-8478-329a28ee4f2e", "id": "d104d9af-fa34-4261-8478-329a28ee4f2e",
"metadata": { "metadata": {
"tags": [] "tags": []
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"# Load csv dataset\n", "def mongoLoadCsv(csvfile):\n",
"data = pd.read_csv(\"out.csv\")" " data = pd.read_csv(\"out.csv\")\n",
" return data\n",
"\n",
"\n",
"def mongoConnect():\n",
" client = MongoClient(\n",
" \"mongodb://{}:{}@{}/EURUSDtest?retryWrites=true&w=majority\".format(\n",
" MongoUser, MongoKey, MongoUrl\n",
" ),\n",
" authSource=\"admin\",\n",
" )\n",
" return client\n",
"\n",
"\n",
"def mongoWriteDict():\n",
" client = mongoConnect()\n",
" db = client[\"EUROUSDtest\"]\n",
" collection = db[\"finance\"]\n",
" # data.reset_index(inplace=True)\n",
" data_dict = data.to_dict(\"records\")\n",
" return data_dict\n",
" collection.insert_many(data_dict)\n",
" return 0"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 127,
"id": "0af8f72c-5b58-4dfc-af36-c5b4bc79f127", "id": "739de6aa-313f-4ccd-96c8-fa22d0cc687e",
"metadata": { "metadata": {
"tags": [] "tags": []
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"# Connect to MongoDB\n", "data = mongoLoadCsv(\"out.csv\")"
"client = MongoClient(\n",
" # \"mongodb://192.168.1.133:27017\"\n",
" \"mongodb://{}:{}@{}/EURUSDtest?retryWrites=true&w=majority\".format(\n",
" MongoUser, MongoKey, MongoUrl\n",
" ),\n",
" authSource=\"admin\",\n",
")"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 128,
"id": "f1b20d15-f5af-463c-813f-ffae61119de1", "id": "0af8f72c-5b58-4dfc-af36-c5b4bc79f127",
"metadata": { "metadata": {
"tags": [] "tags": []
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"db = client[\"EUROUSDtest\"]\n", "start = timeit.default_timer()\n",
"collection = db[\"finance\"]\n", "dfCh = mongoWriteDict()\n",
"# data.reset_index(inplace=True)\n", "stop = timeit.default_timer()\n",
"data_dict = data.to_dict(\"records\")" "mongo_write_execution_time = stop - start"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 129,
"id": "70674d23-f375-4659-87ec-c745dec96d54", "id": "0757f14c-4677-41d3-90d8-63b884e24e7e",
"metadata": { "metadata": {
"tags": [] "tags": []
}, },
"outputs": [], "outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"6.021722518999013\n"
]
}
],
"source": [ "source": [
"%%time\n", "print(mongo_write_execution_time)"
"# Insert collection\n",
"collection.insert_many(data_dict)"
] ]
}, },
{ {
@ -1397,7 +1588,6 @@
"cell_type": "markdown", "cell_type": "markdown",
"id": "97405e42-61dc-42c7-8220-237a312c0ec7", "id": "97405e42-61dc-42c7-8220-237a312c0ec7",
"metadata": { "metadata": {
"jp-MarkdownHeadingCollapsed": true,
"tags": [] "tags": []
}, },
"source": [ "source": [
@ -1406,28 +1596,42 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 130,
"id": "bbcdb883-d6dc-46db-88db-4c90b84522ba", "id": "bbcdb883-d6dc-46db-88db-4c90b84522ba",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"cursor = duckdb.connect()\n", "def duckdbConnect():\n",
"print(cursor.execute(\"SELECT 42\").fetchall())" " cursor = duckdb.connect()\n",
" print(cursor.execute(\"SELECT 42\").fetchall())\n",
" return cursor"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 131,
"id": "35025a6e-9dc7-46cf-a792-76b3d84f1ac0", "id": "35025a6e-9dc7-46cf-a792-76b3d84f1ac0",
"metadata": { "metadata": {
"tags": [] "tags": []
}, },
"outputs": [], "outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"CPU times: user 3.66 ms, sys: 4 ms, total: 7.66 ms\n",
"Wall time: 7.27 ms\n"
]
}
],
"source": [ "source": [
"%%time\n", "%%time\n",
"conn = duckdb.connect()\n", "conn = duckdb.connect()\n",
"data = pd.read_csv(\"out.csv\")\n", "\n",
"conn.register(\"EURUSDtest\", data)" "# in memory???\n",
"def duckdbWrite():\n",
" data = pd.read_csv(\"out.csv\")\n",
" conn.register(\"EURUSDtest\", data)"
] ]
}, },
{ {

Write Preview
Loading…
Cancel
Save