diff --git a/README.md b/README.md index 10e86fa8..e7fb82d2 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,7 @@ proxy.sh [target] [port] - `-s` or `--serve` to start the proxy server - `-t` or `--train` to train the model - Default + Example: ```bash python -m quasar_firewall [options] [target] [port] diff --git a/quasar_project_firewall.egg-info/PKG-INFO b/quasar.egg-info/PKG-INFO similarity index 94% rename from quasar_project_firewall.egg-info/PKG-INFO rename to quasar.egg-info/PKG-INFO index 7c587f7d..c62ac388 100644 --- a/quasar_project_firewall.egg-info/PKG-INFO +++ b/quasar.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 2.1 -Name: quasar-project-firewall -Version: 1.0.1.1 +Name: quasar +Version: 1.0.0 Summary: An Artificial Intelligent firewall that detects malicious HTTP requests Home-page: https://github.com/kyro-dev/Quasar-Project Author: Kyro Dev @@ -43,6 +43,7 @@ Description: # Quasar Firewall - `-s` or `--serve` to start the proxy server - `-t` or `--train` to train the model - Default + Example: ```bash python -m quasar_firewall [options] [target] [port] @@ -70,6 +71,8 @@ Description: # Quasar Firewall Platform: UNKNOWN +Classifier: Intended Audience :: Developers +Classifier: Intended Audience :: Information Technology Classifier: Programming Language :: Python :: 3 Classifier: License :: OSI Approved :: GNU General Public License (GPL) Classifier: Operating System :: OS Independent diff --git a/quasar.egg-info/SOURCES.txt b/quasar.egg-info/SOURCES.txt new file mode 100644 index 00000000..fd935540 --- /dev/null +++ b/quasar.egg-info/SOURCES.txt @@ -0,0 +1,21 @@ +README.md +setup.py +quasar/__init__.py +quasar/__main__.py +quasar/app.py +quasar/dataset.py +quasar/detector.py +quasar/detector_flask.py +quasar/evolution.py +quasar/neuralnet.py +quasar/request.py +quasar/request_parser.py +quasar/visualize.py +quasar.egg-info/PKG-INFO +quasar.egg-info/SOURCES.txt +quasar.egg-info/dependency_links.txt +quasar.egg-info/entry_points.txt +quasar.egg-info/requires.txt +quasar.egg-info/top_level.txt +scripts/proxy.sh +scripts/train.sh \ No newline at end of file diff --git a/quasar_project_firewall.egg-info/dependency_links.txt b/quasar.egg-info/dependency_links.txt similarity index 100% rename from quasar_project_firewall.egg-info/dependency_links.txt rename to quasar.egg-info/dependency_links.txt diff --git a/quasar_project_firewall.egg-info/entry_points.txt b/quasar.egg-info/entry_points.txt similarity index 100% rename from quasar_project_firewall.egg-info/entry_points.txt rename to quasar.egg-info/entry_points.txt diff --git a/quasar_project_firewall.egg-info/requires.txt b/quasar.egg-info/requires.txt similarity index 84% rename from quasar_project_firewall.egg-info/requires.txt rename to quasar.egg-info/requires.txt index f663486f..45ed0920 100644 --- a/quasar_project_firewall.egg-info/requires.txt +++ b/quasar.egg-info/requires.txt @@ -1,12 +1,12 @@ -autopep8==1.5.4 +autopep8 bleach==3.2.1 certifi==2020.12.5 -chardet==4.0.0 +chardet click==7.1.2 colorama==0.4.4 cycler==0.10.0 docutils==0.16 -Flask==1.1.2 +Flask graphviz==0.15 idna==2.10 itsdangerous==1.1.0 @@ -17,7 +17,7 @@ kiwisolver==1.3.1 MarkupSafe==1.1.1 matplotlib==3.3.3 neat-python==0.92 -numpy==1.19.4 +numpy==1.19.3 packaging==20.7 Pillow==8.0.1 pkginfo==1.6.1 @@ -27,7 +27,7 @@ pyparsing==2.4.7 python-dateutil==2.8.1 pywin32-ctypes==0.2.0 readme-renderer==28.0 -requests==2.25.0 +requests requests-toolbelt==0.9.1 rfc3986==1.4.0 scikit-learn==0.23.2 @@ -36,7 +36,7 @@ six==1.15.0 threadpoolctl==2.1.0 toml==0.10.2 tqdm==4.54.1 -twine==3.2.0 urllib3==1.26.2 webencodings==0.5.1 -Werkzeug==1.0.1 +wheel +Werkzeug diff --git a/quasar_project_firewall.egg-info/top_level.txt b/quasar.egg-info/top_level.txt similarity index 100% rename from quasar_project_firewall.egg-info/top_level.txt rename to quasar.egg-info/top_level.txt diff --git a/quasar/__init__.py b/quasar/__init__.py index e69de29b..55635a05 100644 --- a/quasar/__init__.py +++ b/quasar/__init__.py @@ -0,0 +1,9 @@ +""" +Quasar + +A fully artificaially intelligent firewall +""" + +__version__ = "1.0.0" +__author__ = 'Kyro' +__credits__ = 'KyroDev' diff --git a/quasar/__main__.py b/quasar/__main__.py index 4a7ccd46..3829a39f 100644 --- a/quasar/__main__.py +++ b/quasar/__main__.py @@ -10,7 +10,7 @@ parser = argparse.ArgumentParser(description="AI Firewall") parser.add_argument("-t", "--train", help="Train the model", - action="store_true") + nargs='?', const=True) parser.add_argument("-s", "--serve", nargs=2, metavar=("target", "port"), help="Start the proxy server on 'port' with proxy destination being 'target'") diff --git a/quasar/dataset.py b/quasar/dataset.py index 8ddf3262..2a2fcef7 100644 --- a/quasar/dataset.py +++ b/quasar/dataset.py @@ -6,7 +6,14 @@ from .request import Request -def parse_dataset(f:str): +def get_directory(dir_name: str = ''): + from pathlib import Path + + path_to_quasar = Path(__file__).parents[0] + return path_to_quasar / dir_name + + +def parse_dataset(f: str): """ - Parses the dataset - Input dataset must be xml @@ -27,7 +34,7 @@ def parse_dataset(f:str): class_elem = sample.find('class') req = sample.find('request') data = {} - + for item in req: tag = item.tag if tag.lower() not in ["uri", "query", "body"]: @@ -40,7 +47,7 @@ def parse_dataset(f:str): requests.append(Request(data).to_dict()) - file = open("dataset.json", "w") + file = open("./datasets/dataset.json", "w") file.write(json.dumps(requests)) @@ -49,26 +56,27 @@ def parse_dataset(f:str): return requests -def load_dataset(file:str="./datasets/web-application-attacks-datasets/ecml_pkdd/learning_dataset.xml"): +def load_dataset(file: str = f"{get_directory('datasets') / 'web-application-attacks-datasets/ecml_pkdd/learning_dataset.xml'}"): """ Handles dataset loading, returns parsed dataset in `List[Request]` form """ + os.chdir(str(get_directory())) global data data = None reqs: List[Request] = [] - if not os.path.exists("./dataset.json"): + if not os.path.exists("./datasets/dataset.json"): data = parse_dataset(file) try: - with open("./dataset.json") as f: + with open("./datasets/dataset.json") as f: data = json.load(f) for d in data: reqs.append(Request(d)) - except json.decoder.JSONDecodeError: - os.remove('./dataset.json') - raise RuntimeError("An error occured, please try again later") + except json.decoder.JSONDecodeError as e: + import traceback + traceback.print_exc() return reqs diff --git a/quasar/detector_flask.py b/quasar/detector_flask.py index e60e474c..bf45f7ed 100644 --- a/quasar/detector_flask.py +++ b/quasar/detector_flask.py @@ -9,6 +9,13 @@ from .request_parser import BodyParser, QueryParser +def get_directory(dir_name: str): + from pathlib import Path + + path_to_quasar = Path(__file__).parents[0] + return str(path_to_quasar / dir_name) + + class DetectorMiddleware(object): """ Middleware for detecting hacks @@ -17,12 +24,13 @@ class DetectorMiddleware(object): def __init__(self, app: Flask): print("Loading models") self.app = app - self.evolution: Evolution = Evolution.load('499', './models') + self.evolution: Evolution = Evolution.load( + '499', get_directory("models")) self.body_parser = BodyParser.load() self.query_parser = QueryParser.load() self.detector = Detector( self.evolution, self.body_parser, self.query_parser) - + print("Finished loading") try: @@ -44,8 +52,8 @@ def __call__(self, environ, start_response): "is_hack": None }) - valid = self.detector.predict(data, request.get_data(), request.query_string) - + valid = self.detector.predict( + data, request.get_data(), request.query_string) if valid: # Its a hack @@ -53,4 +61,3 @@ def __call__(self, environ, start_response): return res(environ, start_response) return self.app(environ, start_response) - diff --git a/quasar/evolution.py b/quasar/evolution.py index 3d99810b..b1cecd3b 100644 --- a/quasar/evolution.py +++ b/quasar/evolution.py @@ -43,12 +43,15 @@ def train(self, config_path: str): winner = self.p.run(self.__eval_genome__, self.generations) - node_names = {-1: "method", -2: "content-type", -3: "protocol", 0: "Hack probability"} + node_names = {-1: "method", -2: "content-type", - + 3: "protocol", 0: "Hack probability"} visualize.draw_net(self.__get_config__( config_path), winner, view=True, filename="./visualizations/model", node_names=node_names) - visualize.plot_stats(self.stats, filename="./visualizations/avg_fitness.svg", ylog=False, view=True) - visualize.plot_species(self.stats, filename="./visualizations/speciation.svg", view=True) + visualize.plot_stats( + self.stats, filename="./visualizations/avg_fitness.svg", ylog=False, view=True) + visualize.plot_species( + self.stats, filename="./visualizations/speciation.svg", view=True) return winner @@ -68,13 +71,16 @@ def predict(self, input: Request): @staticmethod def load(checkpoint: str, session: str = '.'): + from pathlib import Path """ Loads checkpoint with the prefix of `neat-chekpoint-` - checkpoint: desired checkpoint id - session: session directory (defaults to current directory) """ + path = Path(str(session)) / f"neat-checkpoint-{str(checkpoint)}" + point: Population = neat.Checkpointer.restore_checkpoint( - f'{str(session)}/neat-checkpoint-{str(checkpoint)}') + f'{str(path)}') return Evolution(point) @@ -199,7 +205,7 @@ def train(self, config_path: str): pe = neat.ThreadedEvaluator( multiprocessing.cpu_count() - 1, self.__eval_genome__) - + winner = self.p.run(pe.evaluate, self.generations) node_names = {-1: "method", -2: "headers", -3: "protocol", - @@ -211,7 +217,7 @@ def train(self, config_path: str): visualize.plot_species(self.stats, view=True) return winner - + @staticmethod def load(checkpoint: str, session: str = '.'): """ diff --git a/quasar_project.egg-info/PKG-INFO b/quasar_project.egg-info/PKG-INFO new file mode 100644 index 00000000..d273dddc --- /dev/null +++ b/quasar_project.egg-info/PKG-INFO @@ -0,0 +1,80 @@ +Metadata-Version: 2.1 +Name: quasar-project +Version: 1.0.0 +Summary: An Artificial Intelligent firewall that detects malicious HTTP requests +Home-page: https://github.com/kyro-dev/Quasar-Project +Author: Kyro Dev +Author-email: kyro.captcha@gmail.com +License: UNKNOWN +Description: # Quasar Firewall + An AI powered firewall designed to detect mallicious HTTP requests and decide weather to process them or not. + + ``` + pip install quasar-project-firewall + ``` + + # Usage + ## As Flask Middleware + *Note: Model must be trained before starting proxy server* + ```python + from detector_flask import DetectorMiddleware + ... + app.wsgi_app = DetectorMiddleware(app.wsgi_app) + ... + ``` + ## Starting Proxy From Command Line + *Note: Must have python installed on local machine and added to PATH*
+ *Note: Model must be trained before starting proxy server* + + ### **With Bash** + Call `proxy.sh` with paramaters `target` and `port` + - `target` is the proxy destination + - defaults to `http://localhost:8080` + - `port` is the local port to run the server on + - defaults to 5000 + + Example: + ```bash + proxy.sh [target] [port] + ``` + + ### **As Python Module** + - `options` + - `-s` or `--serve` to start the proxy server + - `-t` or `--train` to train the model + - Default + + Example: + ```bash + python -m quasar_firewall [options] [target] [port] + ``` + + Redirect requests from original server to proxy server + + + ## Training the Model + *Note: Must have python installed on local machine and added to PATH* + + ### **With Bash** + Call `train.sh` + + Example: + ```bash + train.sh + ``` + + ### **As Python Module** + Example: + ```bash + python -m quasar_firewall -t + ``` + + +Platform: UNKNOWN +Classifier: Intended Audience :: Developers +Classifier: Intended Audience :: Information Technology +Classifier: Programming Language :: Python :: 3 +Classifier: License :: OSI Approved :: GNU General Public License (GPL) +Classifier: Operating System :: OS Independent +Requires-Python: >=3.6 +Description-Content-Type: text/markdown diff --git a/quasar_project.egg-info/SOURCES.txt b/quasar_project.egg-info/SOURCES.txt new file mode 100644 index 00000000..c8755293 --- /dev/null +++ b/quasar_project.egg-info/SOURCES.txt @@ -0,0 +1,21 @@ +README.md +setup.py +quasar/__init__.py +quasar/__main__.py +quasar/app.py +quasar/dataset.py +quasar/detector.py +quasar/detector_flask.py +quasar/evolution.py +quasar/neuralnet.py +quasar/request.py +quasar/request_parser.py +quasar/visualize.py +quasar_project.egg-info/PKG-INFO +quasar_project.egg-info/SOURCES.txt +quasar_project.egg-info/dependency_links.txt +quasar_project.egg-info/entry_points.txt +quasar_project.egg-info/requires.txt +quasar_project.egg-info/top_level.txt +scripts/proxy.sh +scripts/train.sh \ No newline at end of file diff --git a/quasar_project.egg-info/dependency_links.txt b/quasar_project.egg-info/dependency_links.txt new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/quasar_project.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/quasar_project.egg-info/entry_points.txt b/quasar_project.egg-info/entry_points.txt new file mode 100644 index 00000000..60093b08 --- /dev/null +++ b/quasar_project.egg-info/entry_points.txt @@ -0,0 +1,3 @@ +[console_scripts] +quasar = quasar.__main__:main + diff --git a/quasar_project.egg-info/requires.txt b/quasar_project.egg-info/requires.txt new file mode 100644 index 00000000..45ed0920 --- /dev/null +++ b/quasar_project.egg-info/requires.txt @@ -0,0 +1,42 @@ +autopep8 +bleach==3.2.1 +certifi==2020.12.5 +chardet +click==7.1.2 +colorama==0.4.4 +cycler==0.10.0 +docutils==0.16 +Flask +graphviz==0.15 +idna==2.10 +itsdangerous==1.1.0 +Jinja2==2.11.2 +joblib==0.17.0 +keyring==21.5.0 +kiwisolver==1.3.1 +MarkupSafe==1.1.1 +matplotlib==3.3.3 +neat-python==0.92 +numpy==1.19.3 +packaging==20.7 +Pillow==8.0.1 +pkginfo==1.6.1 +pycodestyle==2.6.0 +Pygments==2.7.3 +pyparsing==2.4.7 +python-dateutil==2.8.1 +pywin32-ctypes==0.2.0 +readme-renderer==28.0 +requests +requests-toolbelt==0.9.1 +rfc3986==1.4.0 +scikit-learn==0.23.2 +scipy==1.5.4 +six==1.15.0 +threadpoolctl==2.1.0 +toml==0.10.2 +tqdm==4.54.1 +urllib3==1.26.2 +webencodings==0.5.1 +wheel +Werkzeug diff --git a/quasar_project.egg-info/top_level.txt b/quasar_project.egg-info/top_level.txt new file mode 100644 index 00000000..bfeb1fa8 --- /dev/null +++ b/quasar_project.egg-info/top_level.txt @@ -0,0 +1 @@ +quasar diff --git a/quasar_project_firewall.egg-info/SOURCES.txt b/quasar_project_firewall.egg-info/SOURCES.txt deleted file mode 100644 index 8b7194ca..00000000 --- a/quasar_project_firewall.egg-info/SOURCES.txt +++ /dev/null @@ -1,21 +0,0 @@ -README.md -setup.py -quasar/__init__.py -quasar/__main__.py -quasar/app.py -quasar/dataset.py -quasar/detector.py -quasar/detector_flask.py -quasar/evolution.py -quasar/neuralnet.py -quasar/request.py -quasar/request_parser.py -quasar/visualize.py -quasar_project_firewall.egg-info/PKG-INFO -quasar_project_firewall.egg-info/SOURCES.txt -quasar_project_firewall.egg-info/dependency_links.txt -quasar_project_firewall.egg-info/entry_points.txt -quasar_project_firewall.egg-info/requires.txt -quasar_project_firewall.egg-info/top_level.txt -scripts/proxy.sh -scripts/train.sh \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 5f59695f..a3b75846 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,12 +1,12 @@ -autopep8==1.5.4 +autopep8 bleach==3.2.1 certifi==2020.12.5 -chardet==4.0.0 +chardet click==7.1.2 colorama==0.4.4 cycler==0.10.0 docutils==0.16 -Flask==1.1.2 +Flask graphviz==0.15 idna==2.10 itsdangerous==1.1.0 @@ -17,7 +17,7 @@ kiwisolver==1.3.1 MarkupSafe==1.1.1 matplotlib==3.3.3 neat-python==0.92 -numpy==1.19.4 +numpy==1.19.3 packaging==20.7 Pillow==8.0.1 pkginfo==1.6.1 @@ -27,7 +27,7 @@ pyparsing==2.4.7 python-dateutil==2.8.1 pywin32-ctypes==0.2.0 readme-renderer==28.0 -requests==2.25.0 +requests requests-toolbelt==0.9.1 rfc3986==1.4.0 scikit-learn==0.23.2 @@ -36,7 +36,7 @@ six==1.15.0 threadpoolctl==2.1.0 toml==0.10.2 tqdm==4.54.1 -twine==3.2.0 urllib3==1.26.2 webencodings==0.5.1 -Werkzeug==1.0.1 \ No newline at end of file +wheel +Werkzeug \ No newline at end of file diff --git a/scripts/proxy.sh b/scripts/proxy.sh index 8b4dc074..dc711b16 100644 --- a/scripts/proxy.sh +++ b/scripts/proxy.sh @@ -1,2 +1,2 @@ -python main.py -s $0 $1 +python -m quasar -s $0 $1 timeout 10 \ No newline at end of file diff --git a/scripts/train.sh b/scripts/train.sh index 6ccfbaa9..99ab1528 100644 --- a/scripts/train.sh +++ b/scripts/train.sh @@ -1,2 +1,2 @@ -python main.py -t +python -m quasar -t timeout 10 \ No newline at end of file diff --git a/setup.py b/setup.py index 061b444c..fb2bd618 100644 --- a/setup.py +++ b/setup.py @@ -8,8 +8,8 @@ long_description = fh.read() setuptools.setup( - name="quasar-project-firewall", # Replace with your own username - version="1.0.1.1", + name="quasar-project", # Replace with your own username + version="1.0.0", author="Kyro Dev", author_email="kyro.captcha@gmail.com", description="An Artificial Intelligent firewall that detects malicious HTTP requests", @@ -17,10 +17,14 @@ long_description_content_type="text/markdown", url="https://github.com/kyro-dev/Quasar-Project", packages=setuptools.find_packages(), + package_data={'': ['./models/*', './datasets/*', './datasets/web-application-attacks-datasets/ecml_pkdd/*', './config.txt']}, + include_package_data=True, classifiers=[ + "Intended Audience :: Developers", + "Intended Audience :: Information Technology", "Programming Language :: Python :: 3", "License :: OSI Approved :: GNU General Public License (GPL)", - "Operating System :: OS Independent", + "Operating System :: OS Independent" ], install_requires=INSTALL_REQUIRES, python_requires='>=3.6',