fix conda environment, query_user_environment.yml

Example_Data_NB.ipynb

@@ -11,7 +11,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 7,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -33,7 +33,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 8,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -58,20 +58,20 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 9,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Output for symbol DIA:\n",
-      "Errors for symbol DIA:\n"
+      "Output for symbol SPY:\n",
+      "Errors for symbol SPY:\n"
      ]
     }
    ],
    "source": [
-    "symbols = [\"DIA\"]\n",
+    "symbols = [\"SPY\"]\n",
     "start_date = \"2020-01-06\"\n",
     "end_date = \"2020-01-07\"\n",
     "row_limit = \"50\"\n",
@@ -121,6 +121,18 @@
     "ax.plot(data['Time'],data['Trade_Price'],marker='*')"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Sample Ref Data Query\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": []
+  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -574,7 +586,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.8"
+   "version": "3.10.9"
   },
   "vscode": {
    "interpreter": {

data_preprocessing/get_data.py

@@ -112,11 +112,16 @@ def get_quotes(symbols, start_date, end_date, row_limit):
             scp.close()
         ssh.close()
 
+import os
+from dotenv import load_dotenv
+import paramiko
+from scp import SCPClient
+
 def get_ref(symbols, start_date, end_date, row_limit):
-    # load the contents of the .env file into the environment
+    # Load the contents of the .env file into the environment
     load_dotenv()
 
-    # read the credentials from the environment variables
+    # Read the credentials from the environment variables
     host = os.getenv("host")
     server_user = os.getenv("server_user")
     server_password = os.getenv("server_password")
@@ -132,9 +137,12 @@ def get_ref(symbols, start_date, end_date, row_limit):
         ssh.connect(host, username=server_user, password=server_password)
 
         for symbol in symbols:
+            # Get the year from the start_date
+            year = start_date.split("-")[0]
+
             # Execute a command to change directory and list files
             command = f'source root/anaconda3/conda.sh && conda activate query_user && cd TAQNYSE-Clickhouse && cd server_helpers && \
-                python3 refdata_server_helpers.py "{server_user}" "{server_password}" "{symbol}" "{start_date}" "{end_date}" "{row_limit}"'
+                python3 refdata_server_helpers.py "{db_user}" "{db_pass}" "{year}" 1 "{symbol}" "{start_date}" "{end_date}" "{row_limit}"'
             stdin, stdout, stderr = ssh.exec_command(command)
 
             print(f"Output for symbol {symbol}:")
@@ -145,22 +153,20 @@ def get_ref(symbols, start_date, end_date, row_limit):
             for line in stderr:
                 print('... ' + line.strip('\n'))
 
-            # SCPCLient takes a paramiko transport as an argument
+            # SCPClient takes a paramiko transport as an argument
             scp = SCPClient(ssh.get_transport())
 
-            # fetch the remote file 'trade_results.csv' from the directory 'TAQNYSE-Clickhouse'
+            # Fetch the remote file 'trade_results.csv' from the directory 'TAQNYSE-Clickhouse'
             # and save it to the data directory in the pipelines folder
             local_file_path = f'data/ref_{symbol}_{start_date.replace("-", "")}-{end_date.replace("-", "")}.csv'
-        os.makedirs(os.path.dirname(local_file_path), exist_ok=True)
-        local_file_path = f'data/ref_{symbol}_{start_date.replace("-", "")}-{end_date.replace("-", "")}.csv'
-        os.makedirs(os.path.dirname(local_file_path), exist_ok=True)
-        scp.get('TAQNYSE-Clickhouse/refdata_results.csv', local_file_path)
+            os.makedirs(os.path.dirname(local_file_path), exist_ok=True)
+            scp.get('TAQNYSE-Clickhouse/ref_results.csv', local_file_path)
 
     except Exception as e:
         print(f"An error occurred: {e}")
 
     finally:
         if scp is not None:
-            # close the SCP session
+            # Close the SCP session
             scp.close()
-        ssh.close()
+        ssh.close()

environment.yml

@@ -1,61 +1,60 @@
 name: query_user
 
 channels:
-- conda-forge- bioconda  - anaconda
+  - conda-forge
+  - bioconda
+  - anaconda
   - conda-forge
   - defaults
+
 dependencies:
-  - - _libgcc_mutex
-  - - _openmp_mutex
-  - - blas
-  - - bzip2
-  - - ca-certificates
-  - - certifi
-  - - charset-normalizer
-  - - clickhouse-sqlalchemy
-  - - exchange-calendars
-  - - idna
-  - - intel-openmp
-  - - korean_lunar_calendar
-  - - ld_impl_linux-64
-  - - libffi
-  - - libgcc-ng
-  - - libgomp
-  - - libstdcxx-ng
-  - - libuuid
-  - - mkl
-  - - mkl_fft
-  - - mkl_random
-  - - ncurses
-  - - numpy-base
-  - - openssl
-  - - packaging
-  - - pandas_market_calendars
-  - - pip
-  - - pycparser
-  - - pyluach
-  - - pyopenssl
-  - - pyparsing
-  - - pysocks
-  - - python
-  - - python-dateutil
-  - - python_abi
-  - - pytz
-  - - readline
-  - - requests
-  - - setuptools
-  - - six
-  - - sqlite
-  - - tk
-  - - toolz
-  - - tzdata
-  - - tzlocal
-  - - urllib3
-  - - wheel
-  - - xz
-  - - zlib
+  - _libgcc_mutex
+  - blas
+  - bzip2
+  - ca-certificates
+  - certifi
+  - charset-normalizer
+  - clickhouse-sqlalchemy
+  - exchange-calendars
+  - idna
+  - intel-openmp
+  - korean_lunar_calendar
+  - ld_impl_linux-64
+  - libffi
+  - libuuid
+  - mkl
+  - mkl_fft
+  - mkl_random
+  - ncurses
+  - numpy-base
+  - openssl
+  - packaging
+  - pandas_market_calendars
+  - pip
+  - pycparser
+  - pyluach
+  - pyopenssl
+  - pyparsing
+  - pysocks
+  - python
+  - python-dateutil
+  - python_abi
+  - pytz
+  - readline
+  - requests
+  - setuptools
+  - six
+  - sqlite
+  - tk
+  - toolz
+  - tzdata
+  - tzlocal
+  - urllib3
+  - wheel
+  - xz
+  - zlib
   - pip:
-    - brotlipy=
+    - brotlipy
     - cffi
     - clickhouse-driver
     - configobj

query_user_environment.yml

@@ -1,61 +1,60 @@
 name: query_user
 
 channels:
-- conda-forge- bioconda  - anaconda
+  - conda-forge
+  - bioconda
+  - anaconda
   - conda-forge
   - defaults
+
 dependencies:
-  - - _libgcc_mutex
-  - - _openmp_mutex
-  - - blas
-  - - bzip2
-  - - ca-certificates
-  - - certifi
-  - - charset-normalizer
-  - - clickhouse-sqlalchemy
-  - - exchange-calendars
-  - - idna
-  - - intel-openmp
-  - - korean_lunar_calendar
-  - - ld_impl_linux-64
-  - - libffi
-  - - libgcc-ng
-  - - libgomp
-  - - libstdcxx-ng
-  - - libuuid
-  - - mkl
-  - - mkl_fft
-  - - mkl_random
-  - - ncurses
-  - - numpy-base
-  - - openssl
-  - - packaging
-  - - pandas_market_calendars
-  - - pip
-  - - pycparser
-  - - pyluach
-  - - pyopenssl
-  - - pyparsing
-  - - pysocks
-  - - python
-  - - python-dateutil
-  - - python_abi
-  - - pytz
-  - - readline
-  - - requests
-  - - setuptools
-  - - six
-  - - sqlite
-  - - tk
-  - - toolz
-  - - tzdata
-  - - tzlocal
-  - - urllib3
-  - - wheel
-  - - xz
-  - - zlib
+  - _libgcc_mutex
+  - blas
+  - bzip2
+  - ca-certificates
+  - certifi
+  - charset-normalizer
+  - clickhouse-sqlalchemy
+  - exchange-calendars
+  - idna
+  - intel-openmp
+  - korean_lunar_calendar
+  - ld_impl_linux-64
+  - libffi
+  - libuuid
+  - mkl
+  - mkl_fft
+  - mkl_random
+  - ncurses
+  - numpy-base
+  - openssl
+  - packaging
+  - pandas_market_calendars
+  - pip
+  - pycparser
+  - pyluach
+  - pyopenssl
+  - pyparsing
+  - pysocks
+  - python
+  - python-dateutil
+  - python_abi
+  - pytz
+  - readline
+  - requests
+  - setuptools
+  - six
+  - sqlite
+  - tk
+  - toolz
+  - tzdata
+  - tzlocal
+  - urllib3
+  - wheel
+  - xz
+  - zlib
   - pip:
-    - brotlipy=
+    - brotlipy
     - cffi
     - clickhouse-driver
     - configobj