Added train_model.py and made the necessary modifications

.travis.yml

@@ -1,8 +1,8 @@
 language: python
 python:
-  - "3.5"
   - "3.6"
-  - "3.7-dev"
+  - "3.7"
+  - "3.8"
 script:
   - pytest
 notifications:

profanity_check/__init__.py

@@ -1,2 +1,2 @@
 from .profanity_check import predict, predict_prob
-__version__="1.0.3"
+__version__="1.0.5"

profanity_check/data/train_model.py

@@ -0,0 +1,20 @@
+import pandas as pd
+import numpy as np
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.svm import LinearSVC
+from sklearn.calibration import CalibratedClassifierCV
+from joblib import dump, load
+
+data = pd.read_csv("clean_data.csv")
+texts = data["text"].astype(str)
+y = data["is_offensive"]
+
+vectorizer = TfidfVectorizer(stop_words="english", min_df=0.0001)
+X = vectorizer.fit_transform(texts)
+
+model = LinearSVC(class_weight="balanced", dual=False, tol=1e-2, max_iter=1e5)
+cclf = CalibratedClassifierCV(base_estimator=model)
+cclf.fit(X, y)
+
+dump(vectorizer, "vectorizer.joblib")
+dump(cclf, "model.joblib")

requirements.txt

@@ -1,2 +1,6 @@
-scikit-learn>=0.20.2
-joblib==0.14.1
+
+joblib==0.15.1
+numpy==1.19.0
+# Used for the training script
+pandas==1.0.5
+scikit-learn==0.23.1

setup.py

@@ -5,15 +5,15 @@
 
 setuptools.setup(
   name="profanity-check",
-  version="1.0.3",
+  version="1.0.5",
   author="Victor Zhou",
   author_email="[email protected]",
   description="A fast, robust library to check for offensive language in strings.",
   long_description=long_description,
   long_description_content_type="text/markdown",
   url="https://github.com/vzhou842/profanity-check",
   packages=setuptools.find_packages(),
-  install_requires=['joblib>=0.14.1', 'scikit-learn>=0.20.2'],
+  install_requires=['scikit-learn>=0.23.1', 'joblib>=0.15.1'],
   package_data={ 'profanity_check': ['data/model.joblib', 'data/vectorizer.joblib'] },
   classifiers=[
     "Development Status :: 5 - Production/Stable",

tests/test_profanity_check.py

@@ -9,11 +9,11 @@ def test_accuracy():
     'fUcK u',
     'GO TO hElL, you dirty scum',
   ]
-  assert list(predict(texts)) == [0, 0, 0, 1, 1, 1]
+  assert list(predict(texts)) == [0, 1, 0, 1, 1, 1]
 
   probs = predict_prob(texts)
   for i in range(len(probs)):
-    if i < 3:
+    if i == 0 or i == 2:
       assert probs[i] <= 0.5
     else:
       assert probs[i] >= 0.5