Fix outdated scrip code list (#49)

* Add a scheduled GitHub Action for pushing a stock codes file to
Cloudflare R2.
* Update relevant functions to pull files from R2.
* Update copyright year in all files.

.github/workflows/daily-tests.yml

@@ -2,7 +2,7 @@ name: daily-tests
 
 on:
   schedule:
-    - cron: '0 1 * * *'
+    - cron: '10 1 * * *'
 
 jobs:
   build:
@@ -16,15 +16,17 @@ jobs:
     steps:
     - name: Checkout code
       uses: actions/checkout@v4
+
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
+
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
         pip install -r requirements.txt
+
     - name: Test with pytest
       run: |
-        pip install pytest pytest-cov
         pytest --cov=bsedata --no-cov-on-fail --cov-report=xml

.github/workflows/daily-update-scrip-code-list.yml

@@ -0,0 +1,32 @@
+name: update-scrip-code-list
+
+on:
+  schedule:
+    - cron: '20 1 * * *'
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.10'
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r cron/requirements.txt
+
+    - name: Run script to update scrip codes
+      env:
+        CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
+        R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
+        R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
+      run: |
+        python cron/update_scrip_code_list.py

.gitignore

@@ -8,6 +8,7 @@ bsedata/__pycache__/
 __pycache__/
 *.py[cod]
 venv/
+.env
 
 # pytest artifacts
 .pytest_cache/

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2018 - 2023 Shrey Dabhi
+Copyright (c) 2018 - 2024 Shrey Dabhi
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

Pipfile

@@ -0,0 +1,23 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+beautifulsoup4 = "*"
+requests = "*"
+lxml = "*"
+
+[dev-packages]
+boto3 = "==1.34.28"
+pandas = "==2.2.0"
+sphinxext-opengraph = "*"
+sphinx-copybutton = "*"
+pytest-cov = "*"
+autopep8 = "*"
+pytest = "*"
+sphinx = "*"
+furo = "*"
+
+[requires]
+python_version = "3.10"

README.md

@@ -68,7 +68,7 @@ Refer the documentation at <https://bsedata.readthedocs.io/en/latest/chglog.html
 
 MIT License
 
-Copyright (c) 2018 - 2023 Shrey Dabhi
+Copyright (c) 2018 - 2024 Shrey Dabhi
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

bsedata/__init__.py

@@ -2,7 +2,7 @@
 
     MIT License
 
-    Copyright (c) 2018 - 2023 Shrey Dabhi
+    Copyright (c) 2018 - 2024 Shrey Dabhi
 
     Permission is hereby granted, free of charge, to any person obtaining a copy
     of this software and associated documentation files (the "Software"), to deal

bsedata/bse.py

@@ -2,7 +2,7 @@
 
     MIT License
 
-    Copyright (c) 2018 - 2023 Shrey Dabhi
+    Copyright (c) 2018 - 2024 Shrey Dabhi
 
     Permission is hereby granted, free of charge, to any person obtaining a copy
     of this software and associated documentation files (the "Software"), to deal
@@ -73,17 +73,10 @@ def updateScripCodes(self):
 
         :returns: None
         """
-        r = requests.get('https://static.quandl.com/BSE+Descriptions/stocks.txt')
-        stk = {x.split("|")[1][3:]: x.split("|")[0][:-11] for x in r.text.split("\n") if x != '' and x.split("|")[1][:3] == 'BOM'}
-        stk.pop("CODE", None)
-        indices = {x.split("|")[1]: x.split("|")[0] for x in r.text.split("\n") if x != '' and x.split("|")[1][:3] != 'BOM'}
-        indices.pop("CODE", None)
+        r = requests.get('https://pub-87b187a07d9c42109c9e6999439a583f.r2.dev/stk.json')
         f_stk = open('stk.json', 'w+')
-        f_stk.write(json.dumps(stk))
+        f_stk.write(json.dumps(r.json()))
         f_stk.close()
-        f_indices = open('indices.json', 'w+')
-        f_indices.write(json.dumps(indices))
-        f_indices.close()
         return
 
     def getScripCodes(self):

bsedata/exceptions.py

@@ -2,7 +2,7 @@
 
     MIT License
 
-    Copyright (c) 2018 - 2023 Shrey Dabhi
+    Copyright (c) 2018 - 2024 Shrey Dabhi
 
     Permission is hereby granted, free of charge, to any person obtaining a copy
     of this software and associated documentation files (the "Software"), to deal

bsedata/gainers.py

@@ -2,7 +2,7 @@
 
     MIT License
 
-    Copyright (c) 2018 - 2023 Shrey Dabhi
+    Copyright (c) 2018 - 2024 Shrey Dabhi
 
     Permission is hereby granted, free of charge, to any person obtaining a copy
     of this software and associated documentation files (the "Software"), to deal

bsedata/indices.py

@@ -2,7 +2,7 @@
 
     MIT License
 
-    Copyright (c) 2018 - 2023 Shrey Dabhi
+    Copyright (c) 2018 - 2024 Shrey Dabhi
 
     Permission is hereby granted, free of charge, to any person obtaining a copy
     of this software and associated documentation files (the "Software"), to deal

bsedata/losers.py

@@ -2,7 +2,7 @@
 
     MIT License
 
-    Copyright (c) 2018 - 2023 Shrey Dabhi
+    Copyright (c) 2018 - 2024 Shrey Dabhi
 
     Permission is hereby granted, free of charge, to any person obtaining a copy
     of this software and associated documentation files (the "Software"), to deal

bsedata/quote.py

@@ -2,7 +2,7 @@
 
     MIT License
 
-    Copyright (c) 2018 - 2023 Shrey Dabhi
+    Copyright (c) 2018 - 2024 Shrey Dabhi
 
     Permission is hereby granted, free of charge, to any person obtaining a copy
     of this software and associated documentation files (the "Software"), to deal

cron/update_scrip_code_list.py

@@ -0,0 +1,99 @@
+"""
+
+    MIT License
+
+    Copyright (c) 2018 - 2024 Shrey Dabhi
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE.
+
+"""
+
+import warnings
+warnings.filterwarnings("ignore", category=DeprecationWarning) 
+
+import os
+import sys
+import json
+import logging
+import pandas as pd
+
+logging.basicConfig(
+    stream=sys.stdout,
+    format="%(filename)s:%(lineno)d - %(levelname)s - %(message)s",
+    level=logging.INFO,
+)
+
+logging.info("Loading CSV from NASDAQ")
+
+df = pd.read_csv(
+    "https://data.nasdaq.com/api/v3/databases/BSE/metadata?api_key=5jiXMkYN6mW7fVm6ExGo",
+    compression="zip",
+    header=0,
+    sep=",",
+    quotechar='"',
+).drop(columns=["description", "refreshed_at", "from_date", "to_date"])
+
+logging.info("Loaded CSV from NASDAQ")
+
+logging.info("Processing stock codes")
+
+stk_df = df[df.code.astype(str).str.contains("BOM")].reset_index(drop=True)
+
+stk_final = {
+    x["code"].replace("BOM", ""): x["name"].split("EOD")[0].strip().replace("-$", "")
+    for x in stk_df.to_dict(orient="records")
+}
+
+logging.info("Verifying stock codes")
+
+for stk_code in stk_final.keys():
+    assert len(stk_code) == 6
+
+logging.info("Saving stock codes to file")
+
+f_stk = open("stk.json", "w+")
+f_stk.write(json.dumps(stk_final))
+f_stk.close()
+
+logging.info("Processing index codes")
+
+indices_df = df[~df.code.astype(str).str.contains("BOM")].reset_index(drop=True)
+
+indices_final = {x["code"]: x["name"] for x in indices_df.to_dict(orient="records")}
+
+logging.info("Saving index codes to file")
+
+f_indices = open("indices.json", "w+")
+f_indices.write(json.dumps(indices_final))
+f_indices.close()
+
+logging.info("Uploading files to R2")
+
+import boto3
+
+s3 = boto3.client(
+    service_name="s3",
+    endpoint_url=f"https://{os.environ['CLOUDFLARE_ACCOUNT_ID']}.r2.cloudflarestorage.com",
+    aws_access_key_id=os.environ["R2_ACCESS_KEY_ID"],
+    aws_secret_access_key=os.environ["R2_SECRET_ACCESS_KEY"],
+    region_name="apac",
+)
+
+s3.upload_file(Filename="./stk.json", Bucket="bsedata", Key="stk.json")
+s3.upload_file(Filename="./indices.json", Bucket="bsedata", Key="indices.json")

test_bsedata.py

@@ -2,7 +2,7 @@
 
     MIT License
 
-    Copyright (c) 2018 - 2023 Shrey Dabhi
+    Copyright (c) 2018 - 2024 Shrey Dabhi
 
     Permission is hereby granted, free of charge, to any person obtaining a copy
     of this software and associated documentation files (the "Software"), to deal
@@ -38,44 +38,67 @@ def test_str():
 
 
 def test_repr():
-    assert repr(b) == "<BSE: update_codes=True> Driver Class for Bombay Stock Exchange (BSE)"
+    assert (
+        repr(b)
+        == "<BSE: update_codes=True> Driver Class for Bombay Stock Exchange (BSE)"
+    )
 
 
-@pytest.mark.parametrize("scripCode", ['534976', '500116', '512573'])
+@pytest.mark.parametrize("scripCode", ["534976", "500116", "512573"])
 def test_getQuote_valid(scripCode):
     data = b.getQuote(scripCode)
-    if 'priceBand' not in data.keys():
+    if "priceBand" not in data.keys():
         assert len(data) == 24
     else:
         assert len(data) == 27
 
 
 def test_getQuote_invalid_default():
     with pytest.raises(InvalidStockException) as err_info:
-        b.getQuote('513715')
+        b.getQuote("513715")
 
     assert err_info.value.status == "Inactive stock"
 
 
 def test_getQuote_invalid_custom():
     with pytest.raises(InvalidStockException) as err_info:
-        b.getQuote('538936')
+        b.getQuote("538936")
 
     assert err_info.value.status == "Suspended due to Procedural reasons"
 
 
-def test_verifyCode_valid():
-    assert b.verifyScripCode('534976') == 'V-mart Retail Ltd.'
+@pytest.mark.parametrize(
+    "scrip_code,scrip_name",
+    [
+        ("534976", "V-MART RETAIL LTD."),
+        ("542649", "Rail Vikas Nigam Ltd"),
+        ("541557", "Fine Organic Industries Ltd"),
+    ],
+)
+def test_verifyCode_valid(scrip_code, scrip_name):
+    assert b.verifyScripCode(scrip_code) == scrip_name
 
 
 def test_verifyCode_invalid():
-    assert b.verifyScripCode('534980') == None
-
-
-@pytest.mark.parametrize("category",
-                         ["market_cap/broad", "sector_and_industry", "thematics", "strategy", "sustainability",
-                          "volatility", "composite", "government", "corporate", "money_market"])
+    assert b.verifyScripCode("534980") == None
+
+
+@pytest.mark.parametrize(
+    "category",
+    [
+        "market_cap/broad",
+        "sector_and_industry",
+        "thematics",
+        "strategy",
+        "sustainability",
+        "volatility",
+        "composite",
+        "government",
+        "corporate",
+        "money_market",
+    ],
+)
 def test_getIndices(category):
     indices = b.getIndices(category)
-    datetime.datetime.strptime(indices['updatedOn'], '%d %b %Y')
-    assert len(indices['indices']) >= 1
+    datetime.datetime.strptime(indices["updatedOn"], "%d %b %Y")
+    assert len(indices["indices"]) >= 1