validation data sorting and folder cleanup

.gitignore

@@ -4,5 +4,4 @@
 *.pyc
 Messing Around/
 Messing Around/messy_nb.ipynb
-Validation Data/*.txt
-Validation Data/*.jpg
+Validation Data/raw data Tieferbach/*.txt

Validation Data/raw data Tieferbach/sighting_validation_data.ipynb

@@ -104,7 +104,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3.8.13 ('Georg_DT_Slot3')",
+   "display_name": "Python 3.8.13 ('DT_Slot_3')",
    "language": "python",
    "name": "python3"
   },
@@ -123,7 +123,7 @@
   "orig_nbformat": 4,
   "vscode": {
    "interpreter": {
-    "hash": "84fb123bdc47ab647d3782661abcbe80fbb79236dd2f8adf4cef30e8755eb2cd"
+    "hash": "4a28055eb8a3160fa4c7e4fca69770c4e0a1add985300856aa3fcf4ce32a2c48"
    }
   }
  },

Validation Data/raw data Untertweng/consolidate_validation_data_Untertweng.ipynb

@@ -0,0 +1,118 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import numpy as np\n",
+    "import pandas as pd\n",
+    "import matplotlib.pyplot as plt"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "pegel_df = pd.read_csv('AL_Pegel.txt',delimiter=';')\n",
+    "M1_p_df = pd.read_csv('M1_Druck.txt',delimiter=';')\n",
+    "M2_p_df = pd.read_csv('M2_Druck.txt',delimiter=';')\n",
+    "M1_LA_df = pd.read_csv('M1_LA.txt',delimiter=';')\n",
+    "M2_LA_df = pd.read_csv('M2_LA.txt',delimiter=';')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "pegel_df['Timestamp'] = pegel_df['TIMESTAMP UNIX']+pegel_df['TIMESTAMP MS']/1000.\n",
+    "M1_p_df['Timestamp'] = M1_p_df['TIMESTAMP UNIX']+M1_p_df['TIMESTAMP MS']/1000.\n",
+    "M2_p_df['Timestamp'] = M2_p_df['TIMESTAMP UNIX']+M2_p_df['TIMESTAMP MS']/1000.\n",
+    "M1_LA_df['Timestamp'] = M1_LA_df['TIMESTAMP UNIX']+M1_LA_df['TIMESTAMP MS']/1000.\n",
+    "M2_LA_df['Timestamp'] = M2_LA_df['TIMESTAMP UNIX']+M2_LA_df['TIMESTAMP MS']/1000."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "pegel_df.set_index('Timestamp',inplace=True)\n",
+    "M1_p_df.set_index('Timestamp',inplace=True)\n",
+    "M2_p_df.set_index('Timestamp',inplace=True)\n",
+    "M1_LA_df.set_index('Timestamp',inplace=True)\n",
+    "M2_LA_df.set_index('Timestamp',inplace=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "pegel_df.drop(columns=['VARIABLE','TIMESTAMP UNIX', 'TIMESTAMP MS'],inplace=True)\n",
+    "M1_p_df.drop(columns=['VARIABLE','TIMESTAMP UNIX', 'TIMESTAMP MS'],inplace=True)\n",
+    "M2_p_df.drop(columns=['VARIABLE','TIMESTAMP UNIX', 'TIMESTAMP MS'],inplace=True)\n",
+    "M1_LA_df.drop(columns=['VARIABLE','TIMESTAMP UNIX', 'TIMESTAMP MS'],inplace=True)\n",
+    "M2_LA_df.drop(columns=['VARIABLE','TIMESTAMP UNIX', 'TIMESTAMP MS'],inplace=True)\n",
+    "\n",
+    "pegel_df.rename(columns={'VALUE': 'TB-Pegel'},inplace=True)\n",
+    "M1_p_df.rename(columns={'VALUE': 'M1-Druck'},inplace=True)\n",
+    "M2_p_df.rename(columns={'VALUE': 'M2-Druck'},inplace=True)\n",
+    "M1_LA_df.rename(columns={'VALUE': 'M1-LA'},inplace=True)\n",
+    "M2_LA_df.rename(columns={'VALUE': 'M2-LA'},inplace=True)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 26,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "UT_df = pegel_df.join([M1_LA_df,M1_p_df,M2_LA_df,M2_p_df],how='outer').sort_index()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 27,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "UT_df.to_csv('UT_df.csv')"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3.8.13 ('DT_Slot_3')",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.13"
+  },
+  "orig_nbformat": 4,
+  "vscode": {
+   "interpreter": {
+    "hash": "4a28055eb8a3160fa4c7e4fca69770c4e0a1add985300856aa3fcf4ce32a2c48"
+   }
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

Validation Data/read_validation_data_long.ipynb

@@ -1,179 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": 57,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import numpy as np\n",
-    "import pandas as pd\n",
-    "import time\n",
-    "from datetime import datetime\n",
-    "import matplotlib.pyplot as plt"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 58,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "name = 'August_3'"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 59,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "%matplotlib qt5\n",
-    "\n",
-    "def unpack_line(str):\n",
-    "    time_format = \"%d.%m.%Y %H:%M:%S.%f\"\n",
-    "    index_1 = str.find(';')\n",
-    "    index_2 = str.find(';',index_1+1)\n",
-    "    index_3 = str.find(';',index_2+1)\n",
-    "    index_4 = str.find(';',index_3+1)\n",
-    "    index_5 = str.find(';',index_4+1)\n",
-    "    parameter   = str[0:index_1]\n",
-    "    value       = float(str[index_2+1:index_3])\n",
-    "    timestamp   = time.mktime(datetime.strptime(str[index_5+1:-2],time_format).timetuple())\n",
-    "    return parameter,value,timestamp\n",
-    "\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 60,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "M1_LA_df    = pd.DataFrame(columns=['Timestamp','M1-LA'])\n",
-    "M2_LA_df    = pd.DataFrame(columns=['Timestamp','M2-LA'])\n",
-    "M1_Druck_df = pd.DataFrame(columns=['Timestamp','M1-Druck'])\n",
-    "M2_Druck_df = pd.DataFrame(columns=['Timestamp','M2-Druck'])\n",
-    "\n",
-    "\n",
-    "parameter_old    = ''\n",
-    "value_list      = []\n",
-    "timestamp_list  = []\n",
-    "with open(name+'.txt') as txt_file:\n",
-    "    for line in txt_file:\n",
-    "        if line == \"\":\n",
-    "            break\n",
-    "        parameter_new, value_new, timestamp_new = unpack_line(line)\n",
-    "        if parameter_new != parameter_old:\n",
-    "            if 'M1' in parameter_old and 'Stell_Leitapparat' in parameter_old:\n",
-    "                M1_LA_df['Timestamp']       = timestamp_list[:]\n",
-    "                M1_LA_df['M1-LA']           = value_list[:]\n",
-    "            if 'M1' in parameter_old and 'Spiraldruck' in parameter_old:\n",
-    "                M1_Druck_df['Timestamp']    = timestamp_list[:]\n",
-    "                M1_Druck_df['M1-Druck']     = value_list[:]\n",
-    "            if 'M2' in parameter_old and 'Stell_Leitapparat' in parameter_old:\n",
-    "                M2_LA_df['Timestamp']       = timestamp_list[:]\n",
-    "                M2_LA_df['M2-LA']           = value_list[:]\n",
-    "            if 'M2' in parameter_old and 'Spiraldruck' in parameter_old:\n",
-    "                M2_Druck_df['Timestamp']    = timestamp_list[:]\n",
-    "                M2_Druck_df['M2-Druck']     = value_list[:]\n",
-    "            \n",
-    "            value_list      = []\n",
-    "            timestamp_list  = []\n",
-    "            value_list.append(value_new)\n",
-    "            timestamp_list.append(timestamp_new)\n",
-    "\n",
-    "            parameter_old   = parameter_new\n",
-    "        else:\n",
-    "            if value_new != value_list[-1]:\n",
-    "                value_list.append(value_new)\n",
-    "                timestamp_list.append(timestamp_new) \n",
-    "\n",
-    "    if 'M1' in parameter_old and 'Stell_Leitapparat' in parameter_old:\n",
-    "        M1_LA_df['Timestamp']       = timestamp_list[:]\n",
-    "        M1_LA_df['M1-LA']           = value_list[:]\n",
-    "    if 'M1' in parameter_old and 'Spiraldruck' in parameter_old:\n",
-    "        M1_Druck_df['Timestamp']    = timestamp_list[:]\n",
-    "        M1_Druck_df['M1-Druck']     = value_list[:]\n",
-    "    if 'M2' in parameter_old and 'Stell_Leitapparat' in parameter_old:\n",
-    "        M2_LA_df['Timestamp']       = timestamp_list[:]\n",
-    "        M2_LA_df['M2-LA']           = value_list[:]\n",
-    "    if 'M2' in parameter_old and 'Spiraldruck' in parameter_old:\n",
-    "        M2_Druck_df['Timestamp']    = timestamp_list[:]\n",
-    "        M2_Druck_df['M2-Druck']     = value_list[:]\n",
-    "\n",
-    "M1_LA_df.set_index(['Timestamp'],inplace=True)\n",
-    "M1_Druck_df.set_index(['Timestamp'],inplace=True)\n",
-    "M2_LA_df.set_index(['Timestamp'],inplace=True)\n",
-    "M2_Druck_df.set_index(['Timestamp'],inplace=True)\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 61,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "df = M1_LA_df.join([M2_LA_df,M1_Druck_df,M2_Druck_df],how='outer')\n",
-    "df.sort_index(axis=0,inplace=True)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 62,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# t_vec = df.index.to_numpy()\n",
-    "# M1_LA = df['M1-LA'].to_numpy() \n",
-    "# M2_LA = df['M2-LA'].to_numpy() \n",
-    "# M1_p = df['M1-Druck'].to_numpy() \n",
-    "# M2_p = df['M2-Druck'].to_numpy() \n",
-    "# fig1=plt.figure()\n",
-    "# plt.plot(t_vec,M1_LA)\n",
-    "# fig2=plt.figure()\n",
-    "# plt.plot(t_vec,M2_LA)\n",
-    "# fig3=plt.figure()\n",
-    "# plt.plot(t_vec,M1_p)\n",
-    "# fig4=plt.figure()\n",
-    "# plt.plot(t_vec,M2_p)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 63,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "df.to_csv(name+'.csv')"
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3.8.13 ('DT_Slot_3')",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.8.13"
-  },
-  "orig_nbformat": 4,
-  "vscode": {
-   "interpreter": {
-    "hash": "4a28055eb8a3160fa4c7e4fca69770c4e0a1add985300856aa3fcf4ce32a2c48"
-   }
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}