Merge pull request swirlai#1224 from swirlai/DS-1723

Bump cryptography and Django, remove snowflake.

.github/workflows/urls-checker.yml

@@ -37,7 +37,7 @@ jobs:
         retry_count: 3
 
         # A comma separated patterns to exclude during URL checks
-        exclude_patterns: localhost,api,apis,rss,etc,xx,googleapis,hostname,snowflake,graph.microsoft.com,login.microsoftonline.com,my-host.com
+        exclude_patterns: localhost,api,apis,rss,etc,xx,googleapis,hostname,graph.microsoft.com,login.microsoftonline.com,my-host.com
 
         # Exclude these files from the checker
         exclude_files: Swirl.postman_collection.json,docs/googlec95caf0bd4a8c5df.html,docs/Gemfile,docs/Gemfile.lock,docs/_config.yml,tests/,SearchProviders/

SearchProviders/preloaded.json

@@ -1390,36 +1390,6 @@
             "MongoDB"
         ]
     },
-    {
-        "name": "Free Company Records - Snowflake",
-        "active": false,
-        "default": false,
-        "authenticator": "",
-        "connector": "Snowflake",
-        "url": "<snowflake-instance-address>",
-        "query_template": "SELECT {fields} FROM {table} WHERE {field1} ILIKE '%{query_string}%' AND NULLIF(TRIM(founded), '') IS NOT NULL ORDER BY TRY_TO_NUMBER(REGEXP_REPLACE(SPLIT_PART(size, '-', 1), '[^0-9]', '')) DESC;",
-        "post_query_template": {},
-        "http_request_headers": {},
-        "page_fetch_config_json": {},
-        "query_processors": [
-            "AdaptiveQueryProcessor"
-        ],
-        "query_mappings": "fields=*,sort_by_date=founded,table=FREECOMPANYDATASET,field1=name",
-        "result_grouping_field": "",
-        "result_processors": [
-            "MappingResultProcessor",
-            "CosineRelevancyResultProcessor"
-        ],
-        "response_mappings": "",
-        "result_mappings": "title='{name}  ({founded})',body='{name} was founded in {founded} in {country}. It has {size} employees and operates in the {industry} industry.',url='https://{linkedin_url}',date_published=founded,NO_PAYLOAD",
-        "results_per_query": 10,
-        "credentials": "<username>:<password>:FREE_COMPANY_DATASET:COMPUTE_WH",
-        "eval_credentials": "",
-        "tags": [
-            "Company",
-            "Snowflake"
-        ]
-    },
     {
         "name": "Entities - LittleSis.org",
         "active": false,

requirements.txt

@@ -1,8 +1,7 @@
 amqp==5.2.0
 annotated-types==0.6.0
-anyio==4.2.0
+anyio==4.3.0
 asgiref==3.7.2
-asn1crypto==1.5.1
 attrs==23.2.0
 autobahn==23.6.2
 Automat==22.10.0
@@ -13,7 +12,7 @@ bs4==0.0.2
 cachetools==5.3.2
 catalogue==2.0.10
 celery==5.3.6
-certifi==2023.11.17
+certifi==2024.2.2
 cffi==1.16.0
 channels==4.0.0
 channels-redis==4.2.0
@@ -28,35 +27,33 @@ confection==0.1.4
 constantly==23.10.4
 coreapi==2.3.3
 coreschema==0.0.4
-cryptography==41.0.7
+cryptography==42.0.4
 cssselect==1.2.0
 cymem==2.0.8
-daphne==4.0.0
+daphne==4.1.0
 distro==1.9.0
-Django==5.0.1
+Django==5.0.2
 django-celery-beat==2.1.0
 django-environ==0.11.2
 django-rest-swagger==2.2.0
 django-restframework==0.0.1
 django-timezone-field==4.2.3
 djangorestframework==3.14.0
-dnspython==2.5.0
+dnspython==2.6.1
 docutils==0.20.1
 drf-yasg==1.21.7
 elastic-transport==8.12.0
 elasticsearch==8.12.0
-en-core-web-lg @ https://github.com/explosion/spacy-models/releases/download/en_core_web_lg-3.7.1/en_core_web_lg-3.7.1-py3-none-any.whl#sha256=ab70aeb6172cde82508f7739f35ebc9918a3d07debeed637403c8f794ba3d3dc
-filelock==3.13.1
-google-api-core==2.16.1
-google-auth==2.27.0
-google-cloud-bigquery==3.17.1
+google-api-core==2.17.1
+google-auth==2.28.1
+google-cloud-bigquery==3.17.2
 google-cloud-core==2.4.1
 google-crc32c==1.5.0
 google-resumable-media==2.7.0
 googleapis-common-protos==1.62.0
 h11==0.14.0
-httpcore==1.0.2
-httpx==0.26.0
+httpcore==1.0.4
+httpx==0.27.0
 hyperlink==21.0.0
 idna==3.6
 incremental==22.10.0
@@ -68,75 +65,71 @@ jsonpath-ng==1.6.1
 kombu==5.3.5
 langcodes==3.3.0
 lxml==5.1.0
-MarkupSafe==2.1.4
+MarkupSafe==2.1.5
 msal==1.26.0
 msgpack==1.0.7
 murmurhash==1.0.10
 natsort==8.4.0
 nltk==3.8.1
-numpy==1.26.3
-openai==1.6.1
+numpy==1.26.4
+openai==1.12.0
 openapi-codec==1.3.2
 opensearch-py==2.4.2
 oracledb==2.0.1
 packaging==23.2
 pika==1.3.2
-platformdirs==3.11.0
 ply==3.11
 preshed==3.0.9
 prompt-toolkit==3.0.43
-protobuf==4.25.2
+protobuf==4.25.3
 psycopg2==2.9.9
 pyahocorasick==2.0.0
 pyasn1==0.5.1
 pyasn1-modules==0.3.0
 pycparser==2.21
-pydantic==2.6.0
-pydantic_core==2.16.1
+pydantic==2.6.1
+pydantic_core==2.16.2
 PyJWT==2.8.0
-pymongo==4.6.1
-pyOpenSSL==23.3.0
+pymongo==4.6.2
+pyOpenSSL==24.0.0
 python-crontab==3.0.0
 python-dateutil==2.8.2
-pytz==2023.4
+pytz==2024.1
 PyYAML==6.0.1
 readability-lxml==0.8.1
 redis==5.0.1
 regex==2023.12.25
 requests==2.31.0
 rsa==4.9
 service-identity==24.1.0
-setuptools==69.0.3
+setuptools==69.1.0
 simplejson==3.19.2
 six==1.16.0
 smart-open==6.4.0
 sniffio==1.3.0
-snowflake-connector-python==3.7.0
-sortedcontainers==2.4.0
 soupsieve==2.5
-spacy==3.7.2
+spacy==3.7.4
 spacy-legacy==3.0.12
 spacy-loggers==1.0.5
 sqlparse==0.4.4
 srsly==2.4.8
 statistics==1.0.3.5
-textblob==0.17.1
-thinc==8.2.2
+textblob==0.18.0.post0
+thinc==8.2.3
 tika==2.6.0
-tiktoken==0.5.2
-tomlkit==0.12.3
-tqdm==4.66.1
+tiktoken==0.6.0
+tqdm==4.66.2
 Twisted==23.10.0
 txaio==23.1.1
 typer==0.9.0
 typing_extensions==4.9.0
-tzdata==2023.4
+tzdata==2024.1
 uritemplate==4.1.1
-urllib3==2.2.0
+urllib3==2.2.1
 vine==5.1.0
 wasabi==1.1.2
 wcwidth==0.2.13
 weasel==0.3.4
 whitenoise==6.6.0
 xmltodict==0.13.0
-zope.interface==6.1
+zope.interface==6.2

swirl/connectors/__init__.py

@@ -16,7 +16,6 @@
 from swirl.connectors.microsoft_graph import M365SharePointSites
 from swirl.connectors.microsoft_graph import MicrosoftTeams
 from swirl.connectors.mongodb import MongoDB
-from swirl.connectors.snowflake import Snowflake
 from swirl.connectors.oracle import Oracle
 
 # uncomment the line below to enable PostgreSQL

swirl/connectors/db_connector.py

@@ -120,7 +120,7 @@ def validate_query(self, session=None):
             return False
 
         return True
-    
+
     ########################################
 
     def normalize_response(self):
@@ -136,7 +136,7 @@ def normalize_response(self):
         if not self.response:
             # assume the connector took care of it
             return
-        
+
         rows = self.response
 
         trimmed_rows = []
@@ -146,7 +146,7 @@ def normalize_response(self):
             n_field = 0
             if self.column_names:
                 for field in column_names:
-                    # to handle None columns e.g. Snowflake
+                    # to handle None columns
                     if row[n_field]:
                         dict_row[field] = row[n_field]
                     else:
@@ -168,4 +168,3 @@ def normalize_response(self):
         self.retrieved = retrieved
         self.results = trimmed_rows
         return
-

swirl/models.py

@@ -82,8 +82,7 @@ class SearchProvider(models.Model):
         ('M365SharePointSites', 'M365 SharePoint Sites'),
         ('MicrosoftTeams', 'Microsoft Teams'),
         ('MongoDB', 'MongoDB'),
-        ('Oracle','Oracle'),
-        ('Snowflake','Snowflake')
+        ('Oracle','Oracle')
     ]
     connector = models.CharField(max_length=200, default='RequestsGet', choices=CONNECTOR_CHOICES)
     url = models.CharField(max_length=2048, default=str, blank=True)