diff --git a/frontend/package.json b/frontend/package.json index 9c01542..f5554aa 100755 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "frontend", - "version": "0.2.1", + "version": "0.3.1", "private": true, "scripts": { "serve": "vue-cli-service serve", diff --git a/frontend/src/components/Login.vue b/frontend/src/components/Login.vue index e63c81c..a316db7 100644 --- a/frontend/src/components/Login.vue +++ b/frontend/src/components/Login.vue @@ -78,6 +78,12 @@ export default { }, computed: { ...mapGetters({ auth_status: "auth_status" }) + }, + + beforeMount: function() { + if (this.$store.getters["is_authenticated"]) { + this.$router.push("/"); + } } }; diff --git a/frontend/src/components/MultipleResourceInput.vue b/frontend/src/components/MultipleResourceInput.vue index accf386..02d755e 100644 --- a/frontend/src/components/MultipleResourceInput.vue +++ b/frontend/src/components/MultipleResourceInput.vue @@ -124,7 +124,13 @@ export default { preprocess_resources() { if (!this.resource_list_model) return; - let values = this.resource_list_model.trim().split("\n"); + let values = this.resource_list_model + .trim() + .split("\n") + .filter(elem => elem.length > 0); + + values = [...new Set(values)]; + let classified_resources = []; values.forEach(element => { @@ -210,14 +216,10 @@ export default { send() { this.preprocess_resources.forEach(new_resource => { let payload = { - to_server: { - url: "/api/create_resource", - resource_name: new_resource.resource, - resource_type: new_resource.type - }, - mutation: "add_resource" + resource_name: new_resource.resource, + resource_type: new_resource.type }; - this.$store.dispatch("resource_action", payload); + this.$store.dispatch("add_new_resource", payload); }); }, patch_type(resource, type) { @@ -240,12 +242,9 @@ export default { max-height: 500px; overflow-y: auto; overflow-x: hidden; - } .v-btn--bottom:not(.v-btn--absolute) { bottom: 50px; } - - diff --git a/frontend/src/components/ResourceInput.vue b/frontend/src/components/ResourceInput.vue deleted file mode 100644 index effcad1..0000000 --- a/frontend/src/components/ResourceInput.vue +++ /dev/null @@ -1,82 +0,0 @@ - - - diff --git a/frontend/src/components/ResourceListing.vue b/frontend/src/components/ResourceListing.vue index a0cfaf9..0a7f00b 100644 --- a/frontend/src/components/ResourceListing.vue +++ b/frontend/src/components/ResourceListing.vue @@ -90,13 +90,7 @@ @@ -116,12 +110,12 @@ :class="{ selected: selected_resource._id === item._id }" > - - {{ + + + {{ item[headers[1].value] - }} + }} + @@ -144,7 +138,7 @@ -
No resources yet.
+
No resources yet
 @@ -154,7 +148,6 @@ v-if="a_resource_is_selected" :resource="selected_resource" :grid_space="grid_space" - :resource_list="resourceDescription.resource_list" :key="component_key" > @@ -189,7 +182,7 @@ export default { return 0; } }, - resourceDescription: Object, + resourceType: String, headers: Array, grid_space: Number }, @@ -205,10 +198,7 @@ export default { }, computed: { resource_list: function() { - let resources = this.$store.getters.get_resources( - this.resourceDescription.resource_list - ); - + let resources = this.$store.getters.get_resources(this.resourceType); this.resource_count = resources.length; resources = resources.sort(this.sortcriteria); @@ -236,22 +226,6 @@ export default { } }, methods: { - get_resource_list: function() { - let payload = { - to_server: { - url: "/api/get_resources", - type: this.resourceDescription.type, - fields: this.resourceDescription.fields - }, - mutation: "set_resource_list", - mutation_args: { - list_name: this.resourceDescription.resource_list, - list_values: [] - } - }; - this.$store.dispatch("resource_action", payload); - }, - copy_resource_to_json: async function() { await navigator.clipboard.writeText( JSON.stringify(this.selected_resource, null, 2) @@ -290,16 +264,10 @@ export default { remove_resource_with_confirmation: function() { this.remove_resource = false; let payload = { - to_server: { - url: "/api/unlink_resource", - resource_id: this.selected_resource._id - }, - mutation: "remove_resource", - mutation_args: { - list_name: this.resourceDescription.resource_list - } + resource_id: this.selected_resource._id }; - this.$store.dispatch("resource_action", payload); + + this.$store.dispatch("remove_resource", payload); this.selected_resource = {}; }, @@ -309,8 +277,7 @@ export default { tag_shake: function() { let payload = { - resource_id: this.selected_resource._id, - resource_type: this.selected_resource.resource_type + resource_id: this.selected_resource._id }; this.$store.dispatch("update_resource", payload); }, @@ -320,9 +287,6 @@ export default { rerender_component: function() { this.component_key += 1; } - }, - mounted() { - this.get_resource_list(); } }; diff --git a/frontend/src/components/Root.vue b/frontend/src/components/Root.vue index 807c969..224a44c 100644 --- a/frontend/src/components/Root.vue +++ b/frontend/src/components/Root.vue @@ -1,46 +1,23 @@ @@ -291,30 +212,7 @@ export default { show_apikeys: false, show_change_password: false, update_interval: null, - hash_resource_description: { - type: "hash", - resource_list: "hashlist" - }, - ip_resource_description: { - type: "ip", - resource_list: "iplist" - }, - domain_resource_description: { - type: "domain", - resource_list: "domainlist" - }, - email_resource_description: { - type: "email", - resource_list: "emaillist" - }, - username_resource_description: { - type: "username", - resource_list: "usernamelist" - }, - url_resource_description: { - type: "url", - resource_list: "urllist" - }, + ip_table_headers: [ { text: "IP", @@ -387,7 +285,11 @@ export default { }, computed: { - ...mapGetters(["get_opened_project", "is_project_opened", "auth_status"]), + ...mapGetters([ + "get_opened_project", + "is_project_opened", + "is_authenticated" + ]), username: function() { return this.$store.getters["username"]; } @@ -396,14 +298,14 @@ export default { mounted: function() { let self = this; this.update_interval = setInterval(function() { - if (self.auth_status === "success") { + if (self.is_authenticated) { self.$store.dispatch("update").catch(); } }, 10000); }, beforeMount: function() { - if (this.$store.getters["auth_status"] === "") { + if (!this.$store.getters["is_authenticated"]) { this.$router.push("/login"); } } diff --git a/frontend/src/components/StatusBar.vue b/frontend/src/components/StatusBar.vue index 9720586..8f2cc9a 100644 --- a/frontend/src/components/StatusBar.vue +++ b/frontend/src/components/StatusBar.vue @@ -1,17 +1,11 @@ diff --git a/frontend/src/components/templates/pastebin/index.vue b/frontend/src/components/templates/pastebin/index.vue index 3e55fb0..ef57e74 100644 --- a/frontend/src/components/templates/pastebin/index.vue +++ b/frontend/src/components/templates/pastebin/index.vue @@ -12,22 +12,22 @@ - {{ + + {{ paste.title - }} - Untitled paste + }} + + Untitled paste - User: + User:  {{ paste.user }} No user - Paste date: + Paste date:  {{ from_python_time(paste.date) }} @@ -55,18 +55,17 @@ - Link: + Link:  {{ get_paste_link(paste.paste_key) }} + >{{ get_paste_link(paste.paste_key) }} - Expire: + Expire:  {{ paste.expire }} Never diff --git a/frontend/src/components/templates/robtex/index.vue b/frontend/src/components/templates/robtex/index.vue new file mode 100644 index 0000000..9f4a448 --- /dev/null +++ b/frontend/src/components/templates/robtex/index.vue @@ -0,0 +1,193 @@ + + + diff --git a/frontend/src/store/modules/project.js b/frontend/src/store/modules/project.js index 4641264..5c71e35 100644 --- a/frontend/src/store/modules/project.js +++ b/frontend/src/store/modules/project.js @@ -24,6 +24,7 @@ const getters = { const actions = { [SET_PROJECT]: ({ commit, dispatch }, project) => { commit(SET_PROJECT, project); + dispatch("get_resources", null, { root: true }); }, [RESET_PROJECT]: ({ state, commit, dispatch }) => { dispatch("reset_resource_lists", null, { root: true }); diff --git a/frontend/src/store/modules/resourcelist.js b/frontend/src/store/modules/resourcelist.js index 1abc860..11985bb 100644 --- a/frontend/src/store/modules/resourcelist.js +++ b/frontend/src/store/modules/resourcelist.js @@ -6,20 +6,44 @@ import api_call from "../../utils/api"; Vue.use(Vuex); const state = { - iplist: [], - domainlist: [], - emaillist: [], - hashlist: [], - urllist: [], - usernamelist: [] + resources: [] }; const actions = { - resource_action: async function({ commit }, payload) { - await api_call({ ...payload.to_server }).then(resp => { - payload.server_response = resp.data; - commit(payload.mutation, { ...payload }); - }); + get_resources: async function({ commit, getters }) { + let url = "/api/get_resources"; + let project_id = getters.get_opened_project._id; + + await api_call({ url: url, project_id: project_id }) + .then(resp => { + commit("set_resources", resp.data); + }) + .catch(err => console.log(err)); + }, + + add_new_resource: async function({ commit }, payload) { + let url = "/api/create_resource"; + await api_call({ + url: url, + resource_name: payload.resource_name, + resource_type: payload.resource_type + }) + .then(resp => { + commit("set_resources", resp.data); + }) + .catch(err => console.log(err)); + }, + + remove_resource: async function({ commit }, payload) { + let url = "/api/unlink_resource"; + await api_call({ + url: url, + resource_id: payload.resource_id + }) + .then(_ => { + commit("unlink_resource", payload.resource_id); + }) + .catch(err => console.log(err)); }, update: async function({ commit }) { @@ -46,63 +70,41 @@ const actions = { }; const mutations = { - reset_resource_lists: function() { - Object.keys(state).forEach(el => (state[el] = [])); - }, - - set_resource_list: function(commit, payload) { - let resources = []; - payload.server_response.forEach(resource => { - resources.push(resource); + set_resources: function(commit, resources) { + resources.forEach(resource => { + if (state.resources.some(el => el._id === resource._id)) { + return; + } else { + state.resources.push(resource); + } }); - state[payload.mutation_args.list_name] = resources; }, - remove_resource: function(commit, payload) { - let resource_list = payload.mutation_args.list_name; - - state[resource_list] = state[resource_list].filter( - el => el._id !== payload.to_server.resource_id - ); + reset_resource_lists: function() { + state.resources = []; }, - add_resource: function(commit, payload) { - payload.server_response.forEach(resource => { - let list_name = resource.type + "list"; - let new_resource = resource.new_resource; - let resource_list = state[list_name]; - if (resource_list.some(el => el._id === new_resource._id)) { - return; - } - resource_list.push(new_resource); - }); - - return payload.server_response; + unlink_resource: function(commit, resource_id) { + state.resources = state.resources.filter(el => el._id !== resource_id); }, add_update: async function(commit, update) { let url = "/api/get_resource"; let resource_id = update.resource_id; - let resource_type = update.resource_type; - let resource_list = resource_type + "list"; await api_call({ url: url, - resource_id: resource_id, - resource_type: resource_type + resource_id: resource_id }).then(resp => { let resp_as_json = null; + try { resp_as_json = JSON.parse(resp.data); - // state[resource_list] = state[resource_list].filter( - // el => el._id !== resp_as_json._id - // ); - // state[resource_list].push(resp_as_json); } catch (error) { resp_as_json = resp.data; } - let resource = state[resource_list].find(el => el._id === resource_id); + let resource = state.resources.find(el => el._id === resource_id); resource.plugins = resp_as_json.plugins; resource.tags = resp_as_json.tags; }); @@ -110,8 +112,9 @@ const mutations = { }; const getters = { - // reusable data accessors - get_resources: state => resource_list => state[resource_list] + get_resources: state => resource_type => { + return state.resources.filter(elem => elem.resource_type === resource_type); + } }; export default { diff --git a/server/entities/domain.py b/server/entities/domain.py deleted file mode 100755 index edc7f53..0000000 --- a/server/entities/domain.py +++ /dev/null @@ -1,39 +0,0 @@ -import time -import json -import bson - -from server.db import DB -from server.entities.resource_types import ResourceType -from server.entities.resource_base import ResourceBase - - -class Domains: - db = DB("domain") - - @staticmethod - def get_by_name(domain_name): - domain_exists = Domains.db.collection.find_one({"domain": domain_name}) - if domain_exists: - return Domain(domain_exists["_id"]) - - else: - args = { - "canonical_name": domain_name, - "domain": domain_name, - "creation_time": time.time(), - "plugins": [], - "resource_type": "domain", - "tags": [], - } - inserted_one = Domains.db.collection.insert_one(args) - - return Domain(inserted_one.inserted_id) - - @staticmethod - def get_by_id(resource_id): - return Domain(resource_id) - - -class Domain(ResourceBase): - def __init__(self, resource_id): - super().__init__(resource_id, ResourceType.DOMAIN) diff --git a/server/entities/email.py b/server/entities/email.py deleted file mode 100755 index b6c0c6f..0000000 --- a/server/entities/email.py +++ /dev/null @@ -1,40 +0,0 @@ -import time -import json -import bson - -from server.db import DB -from server.entities.resource_types import ResourceType -from server.entities.resource_base import ResourceBase - - -class Emails: - db = DB("email") - - @staticmethod - def get_by_name(email): - existing_email = Emails.db.collection.find_one({"email": email}) - if existing_email: - return Email(existing_email["_id"]) - - else: - args = { - "canonical_name": email, - "email": email, - "creation_time": time.time(), - "domain": email.split("@")[1], - "plugins": [], - "resource_type": "email", - "tags": [], - } - inserted_one = Emails.db.collection.insert_one(args) - - return Email(inserted_one.inserted_id) - - @staticmethod - def get_by_id(resource_id): - return Email(resource_id) - - -class Email(ResourceBase): - def __init__(self, resource_id): - super().__init__(resource_id, ResourceType.EMAIL) diff --git a/server/entities/hash.py b/server/entities/hash.py deleted file mode 100755 index e023042..0000000 --- a/server/entities/hash.py +++ /dev/null @@ -1,47 +0,0 @@ -import time -import json -import bson - -from server.db import DB -from server.entities.resource_types import ResourceType -from server.entities.resource_base import ResourceBase -from server.entities.hash_types import HashType - - -class Hashes: - db = DB("hash") - - @staticmethod - def get_by_name(hash_string): - hash_string = hash_string.lower() - existing_hash = Hashes.db.collection.find_one({"hash": hash_string}) - if existing_hash: - return Hash(existing_hash["_id"]) - - else: - hash_type = HashType.hash_detection(hash_string) - - if hash_type == HashType.UNKNOWN: - raise Exception(f"Oops. Unknown hash type in Hash.py {hash_string}") - args = { - "hash": hash_string, - "creation_time": time.time(), - "hash_type": hash_type.value, - "hash_short": hash_string[:8], - "canonical_name": hash_string[:8], - "resource_type": "hash", - "plugins": [], - "tags": [], - } - inserted_one = Hashes.db.collection.insert_one(args) - - return Hash(inserted_one.inserted_id) - - @staticmethod - def get_by_id(resource_id): - return Hash(resource_id) - - -class Hash(ResourceBase): - def __init__(self, resource_id): - super().__init__(resource_id, ResourceType.HASH) diff --git a/server/entities/ip.py b/server/entities/ip.py deleted file mode 100755 index e68993f..0000000 --- a/server/entities/ip.py +++ /dev/null @@ -1,43 +0,0 @@ -import time -import json -import bson - -from server.db import DB -from server.entities.resource_types import ResourceType -from server.entities.resource_base import ResourceBase - - -class IPs: - db = DB("ip") - - @staticmethod - def get_by_name(ip_address): - if ":" in ip_address: - ip_address = ip_address.split(":")[0] - - ip = IPs.db.collection.find_one({"address": ip_address}) - if ip: - return IP(ip["_id"]) - - else: - args = { - "canonical_name": ip_address, - "resource_type": "ip", - "address": ip_address, - "creation_time": time.time(), - "plugins": [], - "tags": [], - } - - inserted_one = IPs.db.collection.insert_one(args) - - return IP(inserted_one.inserted_id) - - @staticmethod - def get_by_id(resource_id): - return IP(resource_id) - - -class IP(ResourceBase): - def __init__(self, resource_id): - super().__init__(resource_id, ResourceType.IPv4) diff --git a/server/plugins/plugin_base.py b/server/entities/plugin_base.py similarity index 74% rename from server/plugins/plugin_base.py rename to server/entities/plugin_base.py index e4ebbe5..e24aea8 100755 --- a/server/plugins/plugin_base.py +++ b/server/entities/plugin_base.py @@ -1,5 +1,6 @@ import traceback -from server.entities.resource import Resources, ResourceType +from server.entities.resource_types import ResourceType +from server.entities.resource_manager import ResourceManager # TODO: Unused base class. Either deleted it or refactor plugins base class class PluginBase: @@ -24,11 +25,9 @@ def finishing_task(plugin_name, project_id, resource_id, resource_type, result): print(result) - resource_type = ResourceType(resource_type) - resource = Resources.get(resource_id, resource_type) - resource.set_plugin_results( - plugin_name, project_id, resource_id, resource_type, result - ) + resource = ResourceManager.get(resource_id) + resource.set_plugin_results(plugin_name, project_id, result) + except Exception as e: tb1 = traceback.TracebackException.from_exception(e) print("".join(tb1.format())) diff --git a/server/plugins/plugins.py b/server/entities/plugins.py similarity index 96% rename from server/plugins/plugins.py rename to server/entities/plugins.py index 5fdee87..62d2435 100755 --- a/server/plugins/plugins.py +++ b/server/entities/plugins.py @@ -6,7 +6,7 @@ PLUGIN_DIRECTORY = "server/plugins/" PLUGIN_HIERARCHY = "server.plugins" -EXCLUDE_SET = ["plugins.py", "__init__.py", "plugin_base.py", "TEMPLATE.py"] +EXCLUDE_SET = ["__init__.py", "TEMPLATE.py"] def _load_plugins(resource_type, name=None): diff --git a/server/entities/project.py b/server/entities/project.py index 34d190e..c2b99b1 100755 --- a/server/entities/project.py +++ b/server/entities/project.py @@ -4,7 +4,6 @@ import bson from server.db import DB -from server.entities.resource import Resources from server.entities.resource_types import ResourceType from server.entities.update_central import UpdateCentral @@ -60,7 +59,7 @@ def remove_resource(self, resource_id): def add_resource(self, resource): data = { - "resource_id": resource.get_id(), + "resource_id": resource.resource_id, "resource_type": resource.get_type_value(), } @@ -74,21 +73,13 @@ def get_resource(self, resource_id): {"resource_refs": 1}, ) - def get_resources(self, resource_type): + def get_resources(self): result = self.db.collection.find_one( - { - "_id": self.project_id, - "resource_refs.resource_type": resource_type.value, - }, - {"resource_refs": 1}, + {"_id": self.project_id,}, {"resource_refs": 1}, ) if result and "resource_refs" in result.keys(): - return [ - item["resource_id"] - for item in result["resource_refs"] - if item["resource_type"] == resource_type.value - ] + return [item["resource_id"] for item in result["resource_refs"]] else: return [] diff --git a/server/entities/resource.py b/server/entities/resource.py deleted file mode 100755 index dc3f6a6..0000000 --- a/server/entities/resource.py +++ /dev/null @@ -1,63 +0,0 @@ -import bson -import re -import validators - -from enum import Enum - -from server.db import DB -from server.utils.validations import HashType, hash_detection - -from server.entities.ip import IPs -from server.entities.domain import Domains -from server.entities.email import Emails -from server.entities.hash import Hashes -from server.entities.url import URLs -from server.entities.username import Usernames - -from server.entities.resource_types import ResourceType, ResourceTypeException - - -class Resources: - @staticmethod - def get(resource_field, resource_type=None, get_by_name=False): - if not get_by_name or not resource_type: - attribute = "get_by_id" - else: - attribute = "get_by_name" - - if resource_type == ResourceType.UNKNOWN: - raise ResourceTypeException() - - elif resource_type == ResourceType.IPv4: - return getattr(IPs, attribute)(resource_field) - - elif resource_type == ResourceType.DOMAIN: - return getattr(Domains, attribute)(resource_field) - - elif resource_type == ResourceType.EMAIL: - return getattr(Emails, attribute)(resource_field) - - elif resource_type == ResourceType.HASH: - return getattr(Hashes, attribute)(resource_field) - - elif resource_type == ResourceType.URL: - return getattr(URLs, attribute)(resource_field) - - elif resource_type == ResourceType.USERNAME: - return getattr(Usernames, attribute)(resource_field) - - @staticmethod - def get_data_from_resources(resource_refs): - result = [] - if not resource_refs: - return result - - for resource_ref in resource_refs: - resource = Resources.get( - resource_ref["resource_id"], - ResourceType.get_type_from_string(resource_ref["resource_type"]), - ) - - result.append(resource.get_data()) - - return result diff --git a/server/entities/resource_base.py b/server/entities/resource_base.py index 8162628..2adbaae 100755 --- a/server/entities/resource_base.py +++ b/server/entities/resource_base.py @@ -2,34 +2,135 @@ import json import bson import time +import urllib.parse + from server.db import DB -from server.plugins.plugins import Plugins +from server.entities.plugins import Plugins from server.entities.update_central import UpdateCentral +from server.entities.resource_types import ResourceType +from server.entities.hash_types import HashType + +COLLECTION = "resources" + +# TODO: Legacy method for old database resources +# TODO: Get rid of this legacy method +def get_resource_legacy_method(resource_id): + """ + Lookup the resource_id in all old documents + Returns resource and doc name to change global COLLECTION + """ + docs = ["ip", "url", "username", "hash", "email", "domain"] + + for doc in docs: + collection = DB(doc).collection + resource = collection.find_one({"_id": resource_id}) + + if resource: + return (resource, doc) + + print(f"[resource_base/get_resource_legacy_method]: {resource_id}") + + return (None, None) + + +def enrich_by_type(args): + resource_type = ResourceType(args["resource_type"]) + + if resource_type == ResourceType.IPv4: + args["address"] = args["canonical_name"] + + elif resource_type == ResourceType.USERNAME: + args["username"] = args["canonical_name"] + + elif resource_type == ResourceType.URL: + args["full_url"] = args["canonical_name"] + url_parts = urllib.parse.urlparse(args["full_url"]) + args["scheme"] = url_parts.scheme + args["netloc"] = url_parts.netloc + args["path"] = url_parts.path + args["params"] = url_parts.params + args["query"] = url_parts.query + args["fragment"] = url_parts.fragment -class ResourceBase: - def __init__(self, resource_id, resource_type): + elif resource_type == ResourceType.HASH: + args["hash"] = args["canonical_name"] + args["hash_type"] = HashType.hash_detection(args["hash"]).value + # canonical_name == printable name in the view + args["canonical_name"] = args["hash"][:8] + + elif resource_type == ResourceType.EMAIL: + args["email"] = args["canonical_name"] + if "@" in args["email"]: + args["domain"] = args["email"].split("@")[1] + else: + args["domain"] = None + + elif resource_type == ResourceType.DOMAIN: + args["domain"] = args["canonical_name"] + + else: + print( + f"[entities/resource_base/enrich_by_type]: Unknown resource type {args['resource_type']} when creating resource." + ) + + return args + + +class Resource: + @staticmethod + def collection(collection=COLLECTION): + return DB(collection).collection + + @staticmethod + def create(name, resource_type): + """ + name: name of the resource + type: type as ResourceType + """ + args = { + "canonical_name": name, + "resource_type": resource_type.value, + "creation_time": time.time(), + "plugins": [], + "tags": [], + } + + args = enrich_by_type(args) + result = Resource.collection().insert_one(args) + print(f"Creating new resource with {args}") + return Resource(str(result.inserted_id)) + + def __init__(self, resource_id): self.resource_id = bson.ObjectId(resource_id) - self.type = resource_type + collection = COLLECTION + self.resource = Resource.collection(collection).find_one( + {"_id": self.resource_id} + ) + # TODO: Get rid of this legacy method + # We have found anything, try legacy database method + if not self.resource: + # If found, change global resource database + self.resource, collection = get_resource_legacy_method(self.resource_id) - def collection(self): - return DB(self.get_type_value()).collection + # Store collection name + self.own_collection = collection - def get_id(self): - return self.resource_id + def get_collection(self): + return Resource.collection(self.own_collection) def get_id_as_string(self): return str(self.resource_id) def get_type(self): - return self.type + return ResourceType.get_type_from_string(self.resource["resource_type"]) def get_type_value(self): - return self.type.value + return self.get_type().value def get_data(self): - return DB(self.get_type_value()).collection.find_one({"_id": self.resource_id}) + return self.resource def launch_plugins(self, project_id, profile=None): try: @@ -47,16 +148,14 @@ def launch_plugin(self, project_id, plugin_name, profile=None): tb1 = traceback.TracebackException.from_exception(e) print("".join(tb1.format())) - def set_plugin_results( - self, plugin_name, project_id, resource_id, resource_type, query_result - ): + def set_plugin_results(self, plugin_name, project_id, query_result): # TODO: Should we store an time-based diff of results if they differ? - result_exists = self.collection().find_one( + result_exists = self.get_collection().find_one( {"_id": self.resource_id, f"plugins.name": plugin_name} ) if not result_exists: - self.collection().update_one( + self.get_collection().update_one( {"_id": self.resource_id}, { "$addToSet": { @@ -76,10 +175,10 @@ def set_plugin_results( plugin["results"] = query_result plugin["update_time"] = time.time() - self.collection().replace_one({"_id": self.resource_id}, result_exists) + self.get_collection().replace_one({"_id": self.resource_id}, result_exists) UpdateCentral().set_pending_update( - project_id, resource_id, resource_type, plugin_name + project_id, self.get_id_as_string(), self.get_type(), plugin_name ) def get_plugins(self, project_id): @@ -92,7 +191,7 @@ def get_plugins(self, project_id): def manage_tag(self, tag): try: - resource = self.collection().find_one({"_id": self.get_id()}) + resource = self.get_collection().find_one({"_id": self.resource_id}) if "tags" in resource: if tag["name"] in [t["name"] for t in resource["tags"]]: @@ -106,7 +205,7 @@ def manage_tag(self, tag): else: resource["tags"] = [tag] - self.collection().replace_one({"_id": self.resource_id}, resource) + self.get_collection().replace_one({"_id": self.resource_id}, resource) except Exception as e: tb1 = traceback.TracebackException.from_exception(e) @@ -125,9 +224,10 @@ def get_doc_if_reference(plugin_name, entry): ) return entry - doc = DB(self.get_type_value()).collection.find_one({"_id": self.resource_id}) + # doc = self.get_collection().find_one({"_id": self.resource_id}) # HACK: Curiously, this seens to work in order to eliminated the double "" JSON encoding # when converting from ObjectId to String + doc = self.resource for plugin in doc["plugins"]: if "results" in plugin and isinstance(plugin["results"], list): diff --git a/server/entities/resource_manager.py b/server/entities/resource_manager.py new file mode 100644 index 0000000..879f1d7 --- /dev/null +++ b/server/entities/resource_manager.py @@ -0,0 +1,70 @@ +import bson +import re +import validators + +from enum import Enum + +from server.db import DB +from server.entities.resource_base import Resource +from server.entities.resource_types import ResourceType, ResourceTypeException + + +class ResourceManager: + @staticmethod + def get(resource_id): + """ + Return a resource by its ID. + The resource must exists in database, otherwise use legacy db method + """ + return Resource(resource_id) + + # TODO: "resource_type" should not be needed here, all resources must have a "searchable" field. + @staticmethod + def get_by_name(resource_name, resource_type): + """ + Lookup database for a resource with name "resource_name" + if None is found, then create the resource. + """ + search = "canonical_name" + resource_type = ResourceType.get_type_from_string(resource_type) + + if resource_type == ResourceType.HASH: + search = "hash" + + db = Resource.collection() + result = db.find_one({search: resource_name}) + if result: + return Resource(result["_id"]) + + # TODO: Legacy method for old database resources + # TODO: Get rid of this legacy method + if not result: + docs = ["ip", "url", "username", "hash", "email", "domain"] + for doc in docs: + result = DB(doc).collection.find_one({search: resource_name}) + if result: + return Resource(result["_id"]) + + print(f"Tried get_by_name nothing found {result}") + return None + + @staticmethod + def get_or_create(resource_name, resource_type): + created = False + resource = ResourceManager.get_by_name(resource_name, resource_type) + if not resource: + resource = Resource.create(resource_name, resource_type) + created = True + return (resource, created) + + @staticmethod + def get_data_from_resources(resource_refs): + result = [] + if not resource_refs: + return result + + for resource_ref in resource_refs: + resource = ResourceManager.get(resource_ref["resource_id"],) + result.append(resource.get_data()) + + return result diff --git a/server/entities/url.py b/server/entities/url.py deleted file mode 100755 index 66c94b6..0000000 --- a/server/entities/url.py +++ /dev/null @@ -1,49 +0,0 @@ -import time -import json -import bson -import urllib.parse - -from server.db import DB -from server.entities.resource_types import ResourceType -from server.entities.resource_base import ResourceBase - - -class URLs: - db = DB("url") - - @staticmethod - def get_by_name(url): - previous_url = URLs.db.collection.find_one({"full_url": url}) - if previous_url: - return URL(previous_url["_id"]) - - else: - url_parts = urllib.parse.urlparse(url) - - args = { - "canonical_name": url, - "resource_type": "url", - "scheme": url_parts.scheme, - "netloc": url_parts.netloc, - "path": url_parts.path, - "params": url_parts.params, - "query": url_parts.query, - "fragment": url_parts.fragment, - "full_url": url, - "creation_time": time.time(), - "plugins": [], - "tags": [], - } - - inserted_one = URLs.db.collection.insert_one(args) - - return URL(inserted_one.inserted_id) - - @staticmethod - def get_by_id(resource_id): - return URL(resource_id) - - -class URL(ResourceBase): - def __init__(self, resource_id): - super().__init__(resource_id, ResourceType.URL) diff --git a/server/entities/username.py b/server/entities/username.py deleted file mode 100755 index 20ee267..0000000 --- a/server/entities/username.py +++ /dev/null @@ -1,40 +0,0 @@ -import time -import json -import bson - -from server.db import DB -from server.entities.resource_types import ResourceType -from server.entities.resource_base import ResourceBase - - -class Usernames: - db = DB("username") - - @staticmethod - def get_by_name(username): - existing_username = Usernames.db.collection.find_one({"username": username}) - if existing_username: - return Username(existing_username["_id"]) - - else: - args = { - "canonical_name": username, - "username": username, - "creation_time": time.time(), - "plugins": [], - "resource_type": "username", - "tags": [], - "sites": [], - } - inserted_one = Usernames.db.collection.insert_one(args) - - return Username(inserted_one.inserted_id) - - @staticmethod - def get_by_id(resource_id): - return Username(resource_id) - - -class Username(ResourceBase): - def __init__(self, resource_id): - super().__init__(resource_id, ResourceType.USERNAME) diff --git a/server/plugins/TEMPLATE.py b/server/plugins/TEMPLATE.py index add03c2..876bae2 100644 --- a/server/plugins/TEMPLATE.py +++ b/server/plugins/TEMPLATE.py @@ -11,8 +11,8 @@ # [DONOTDELETE] - Internal usage dependencies -from server.entities.resource import Resources, ResourceType -from server.plugins.plugin_base import finishing_task +from server.entities.resource_types import ResourceType +from server.entities.plugin_base import finishing_task from tasks.tasks import celery_app diff --git a/server/plugins/abuseipdb.py b/server/plugins/abuseipdb.py index b7fc4c2..73deda1 100755 --- a/server/plugins/abuseipdb.py +++ b/server/plugins/abuseipdb.py @@ -3,9 +3,9 @@ import requests from tasks.api_keys import KeyRing -from server.entities.resource import Resources, ResourceType +from server.entities.resource_types import ResourceType from tasks.tasks import celery_app -from server.plugins.plugin_base import finishing_task +from server.entities.plugin_base import finishing_task API_KEY = KeyRing().get("abuseipdb") URL = "https://api.abuseipdb.com/api/v2/check" diff --git a/server/plugins/basic_ip.py b/server/plugins/basic_ip.py index 67bd37d..7bc8560 100755 --- a/server/plugins/basic_ip.py +++ b/server/plugins/basic_ip.py @@ -4,10 +4,10 @@ import traceback from ipwhois import IPWhois -from server.entities.resource import Resources, ResourceType +from server.entities.resource_types import ResourceType from tasks.tasks import celery_app from dns import resolver, reversename -from server.plugins.plugin_base import finishing_task +from server.entities.plugin_base import finishing_task # Which resources are this plugin able to work with diff --git a/server/plugins/binaryedge.py b/server/plugins/binaryedge.py index 21152d4..d7a05e1 100755 --- a/server/plugins/binaryedge.py +++ b/server/plugins/binaryedge.py @@ -3,9 +3,9 @@ import requests from tasks.api_keys import KeyRing -from server.entities.resource import Resources, ResourceType +from server.entities.resource_types import ResourceType from tasks.tasks import celery_app -from server.plugins.plugin_base import finishing_task +from server.entities.plugin_base import finishing_task # 250 requests left, 31 days until renewal. API_KEY = KeyRing().get("binaryedge") diff --git a/server/plugins/botscout.py b/server/plugins/botscout.py index fbb7b34..49b9887 100755 --- a/server/plugins/botscout.py +++ b/server/plugins/botscout.py @@ -1,7 +1,8 @@ import traceback -from server.entities.resource import Resources, ResourceType +from server.entities.resource_types import ResourceType + from tasks.tasks import celery_app -from server.plugins.plugin_base import finishing_task +from server.entities.plugin_base import finishing_task # Login with the following information: # Login Name: zigineki@getnada.com @@ -12,6 +13,7 @@ import json import urllib.request from bs4 import BeautifulSoup +from server.entities.resource_manager import ResourceManager from tasks.api_keys import KeyRing @@ -109,7 +111,7 @@ def botscout_task(plugin_name, project_id, resource_id, resource_type, ip): else: print("BotScout resource type does not found") - resource = Resources.get(resource_id, resource_type) + resource = ResourceManager.get(resource_id) resource.set_plugin_results( plugin_name, project_id, resource_id, resource_type, query_result ) diff --git a/server/plugins/diario.py b/server/plugins/diario.py index 93c6180..925d725 100755 --- a/server/plugins/diario.py +++ b/server/plugins/diario.py @@ -4,9 +4,9 @@ from diario import Diario -from server.plugins.plugin_base import finishing_task +from server.entities.plugin_base import finishing_task from tasks.api_keys import KeyRing -from server.entities.resource import Resources, ResourceType +from server.entities.resource_types import ResourceType from tasks.tasks import celery_app APP_ID = KeyRing().get("diario-appid") diff --git a/server/plugins/dns.py b/server/plugins/dns.py index 58d557a..7cc7323 100755 --- a/server/plugins/dns.py +++ b/server/plugins/dns.py @@ -6,9 +6,9 @@ import dns.resolver as resolver -from server.entities.resource import Resources, ResourceType +from server.entities.resource_types import ResourceType from tasks.tasks import celery_app -from server.plugins.plugin_base import finishing_task +from server.entities.plugin_base import finishing_task # Which resources are this plugin able to work with RESOURCE_TARGET = [ResourceType.DOMAIN] diff --git a/server/plugins/emailrep.py b/server/plugins/emailrep.py index 6e02171..d824d11 100755 --- a/server/plugins/emailrep.py +++ b/server/plugins/emailrep.py @@ -4,8 +4,8 @@ from tasks.api_keys import KeyRing from tasks.tasks import celery_app -from server.entities.resource import Resources, ResourceType -from server.plugins.plugin_base import finishing_task +from server.entities.resource_types import ResourceType +from server.entities.plugin_base import finishing_task # At this time there is no need for an APIKEY # API_KEY = KeyRing().get("emailrep") diff --git a/server/plugins/geoip.py b/server/plugins/geoip.py index 0d4e19c..c623d1c 100755 --- a/server/plugins/geoip.py +++ b/server/plugins/geoip.py @@ -4,9 +4,9 @@ import urllib.request from tasks.api_keys import KeyRing -from server.entities.resource import Resources, ResourceType +from server.entities.resource_types import ResourceType from tasks.tasks import celery_app -from server.plugins.plugin_base import finishing_task +from server.entities.plugin_base import finishing_task # Which resources are this plugin able to work with RESOURCE_TARGET = [ResourceType.IPv4] diff --git a/server/plugins/haveibeenpwned.py b/server/plugins/haveibeenpwned.py index 7d6d02b..c28e53e 100755 --- a/server/plugins/haveibeenpwned.py +++ b/server/plugins/haveibeenpwned.py @@ -1,12 +1,12 @@ import traceback -from server.entities.resource import Resources, ResourceType +from server.entities.resource_types import ResourceType from tasks.tasks import celery_app import json import requests from tasks.api_keys import KeyRing -from server.plugins.plugin_base import finishing_task +from server.entities.plugin_base import finishing_task API_KEY = KeyRing().get("haveibeenpwned") URL = "https://haveibeenpwned.com/api/v3/{service}/{account}" diff --git a/server/plugins/hunterio.py b/server/plugins/hunterio.py index 5ef9450..2a2ba3b 100755 --- a/server/plugins/hunterio.py +++ b/server/plugins/hunterio.py @@ -3,9 +3,9 @@ import requests from tasks.api_keys import KeyRing -from server.entities.resource import Resources, ResourceType +from server.entities.resource_types import ResourceType from tasks.tasks import celery_app -from server.plugins.plugin_base import finishing_task +from server.entities.plugin_base import finishing_task API_KEY = KeyRing().get("hunterio") diff --git a/server/plugins/maltiverse.py b/server/plugins/maltiverse.py index 6b6461d..bb8d96a 100755 --- a/server/plugins/maltiverse.py +++ b/server/plugins/maltiverse.py @@ -7,9 +7,9 @@ from tasks.deps.maltiverse import Maltiverse from tasks.api_keys import KeyRing -from server.entities.resource import Resources, ResourceType +from server.entities.resource_types import ResourceType from tasks.tasks import celery_app -from server.plugins.plugin_base import finishing_task +from server.entities.plugin_base import finishing_task # Maltiverse does not need api key in some operations # Keeping this if they change their mind in a future diff --git a/server/plugins/metagoofil.py b/server/plugins/metagoofil.py index 06dac25..4e3b649 100644 --- a/server/plugins/metagoofil.py +++ b/server/plugins/metagoofil.py @@ -6,7 +6,7 @@ from server.entities.resource_types import ResourceType from tasks.tasks import celery_app -from server.plugins.plugin_base import finishing_task +from server.entities.plugin_base import finishing_task # Which resources are this plugin able to work with RESOURCE_TARGET = [ResourceType.DOMAIN] diff --git a/server/plugins/onyphe.py b/server/plugins/onyphe.py index 4bdb7f7..bca5b6e 100755 --- a/server/plugins/onyphe.py +++ b/server/plugins/onyphe.py @@ -4,9 +4,9 @@ import urllib.request from tasks.api_keys import KeyRing -from server.entities.resource import Resources, ResourceType +from server.entities.resource_types import ResourceType from tasks.tasks import celery_app -from server.plugins.plugin_base import finishing_task +from server.entities.plugin_base import finishing_task API_KEY = KeyRing().get("onyphe") diff --git a/server/plugins/otx.py b/server/plugins/otx.py index a1ec62f..9c9166d 100755 --- a/server/plugins/otx.py +++ b/server/plugins/otx.py @@ -5,6 +5,8 @@ import json import requests, base64 +from server.entities.resource_manager import ResourceManager + from tasks.api_keys import KeyRing # https://otx.alienvault.com/api/v1/indicators/file/6c5360d41bd2b14b1565f5b18e5c203cf512e493/analysis @@ -19,7 +21,7 @@ URL_IPv6 = "https://otx.alienvault.com/api/v1/indicators/IPv6/{ip}/{section}" -from server.entities.resource import Resources, ResourceType +from server.entities.resource_types import ResourceType from tasks.tasks import celery_app @@ -241,7 +243,7 @@ def otx_task(plugin_name, project_id, resource_id, resource_type, target): else: print("OTX resource type does not found") - resource = Resources.get(resource_id, resource_type) + resource = ResourceManager.get(resource_id) resource.set_plugin_results( plugin_name, project_id, resource_id, resource_type, query_result ) diff --git a/server/plugins/pastebin.py b/server/plugins/pastebin.py index 9fc27b7..f45d731 100755 --- a/server/plugins/pastebin.py +++ b/server/plugins/pastebin.py @@ -6,10 +6,10 @@ from server.db import DB from server.entities.pastebin_manager import PastebinManager, Paste -from server.entities.resource import Resources, ResourceType -from server.plugins.plugin_base import finishing_task +from server.entities.resource_types import ResourceType +from server.entities.plugin_base import finishing_task from tasks.api_keys import KeyRing -from tasks.subtasks.googlesearch import restricted_googlesearch +from tasks.googlesearch import restricted_googlesearch from tasks.tasks import celery_app diff --git a/server/plugins/phishtank.py b/server/plugins/phishtank.py index db15188..e202b2c 100755 --- a/server/plugins/phishtank.py +++ b/server/plugins/phishtank.py @@ -6,9 +6,9 @@ from urllib.parse import urlparse from tasks.api_keys import KeyRing -from server.entities.resource import Resources, ResourceType +from server.entities.resource_types import ResourceType from tasks.tasks import celery_app -from server.plugins.plugin_base import finishing_task +from server.entities.plugin_base import finishing_task API_KEY = KeyRing().get("phishtank") diff --git a/server/plugins/pulsedive.py b/server/plugins/pulsedive.py index 4596afe..8de5b92 100644 --- a/server/plugins/pulsedive.py +++ b/server/plugins/pulsedive.py @@ -7,8 +7,8 @@ import requests, base64 from tasks.api_keys import KeyRing - -from server.entities.resource import Resources, ResourceType +from server.entities.resource_manager import ResourceManager +from server.entities.resource_types import ResourceType from tasks.tasks import celery_app API_KEY = KeyRing().get("pulsedive") @@ -136,7 +136,7 @@ def pulsedive_task(plugin_name, project_id, resource_id, resource_type, domain_o else: print("PulseDive resource type does not found") - resource = Resources.get(resource_id, resource_type) + resource = ResourceManager.get(resource_id, resource_type) resource.set_plugin_results( plugin_name, project_id, resource_id, resource_type, query_result ) diff --git a/server/plugins/robtex.py b/server/plugins/robtex.py index e128e31..8d64056 100755 --- a/server/plugins/robtex.py +++ b/server/plugins/robtex.py @@ -2,9 +2,9 @@ import json import requests -from server.entities.resource import Resources, ResourceType +from server.entities.resource_types import ResourceType from tasks.tasks import celery_app -from server.plugins.plugin_base import finishing_task +from server.entities.plugin_base import finishing_task # Which resources are this plugin able to work with RESOURCE_TARGET = [ResourceType.IPv4] diff --git a/server/plugins/sherlock.py b/server/plugins/sherlock.py index 0451e84..9b2d2bb 100755 --- a/server/plugins/sherlock.py +++ b/server/plugins/sherlock.py @@ -4,9 +4,9 @@ import tasks.deps.sherlock.sherlock as _sherlock -from server.entities.resource import Resources, ResourceType +from server.entities.resource_types import ResourceType from tasks.tasks import celery_app -from server.plugins.plugin_base import finishing_task +from server.entities.plugin_base import finishing_task # Which resources are this plugin able to work with RESOURCE_TARGET = [ResourceType.USERNAME] diff --git a/server/plugins/shodan.py b/server/plugins/shodan.py index 0866e76..956557b 100755 --- a/server/plugins/shodan.py +++ b/server/plugins/shodan.py @@ -3,9 +3,9 @@ import requests from tasks.api_keys import KeyRing -from server.entities.resource import Resources, ResourceType +from server.entities.resource_types import ResourceType from tasks.tasks import celery_app -from server.plugins.plugin_base import finishing_task +from server.entities.plugin_base import finishing_task # Which resources are this plugin able to work with RESOURCE_TARGET = [ResourceType.IPv4] diff --git a/server/plugins/tacyt.py b/server/plugins/tacyt.py index ad18a5d..96baefd 100755 --- a/server/plugins/tacyt.py +++ b/server/plugins/tacyt.py @@ -1,10 +1,10 @@ import traceback -from server.entities.resource import Resources, ResourceType +from server.entities.resource_types import ResourceType from tasks.tasks import celery_app from tasks.deps.tacyt import TacytApp as tacytsdk from tasks.api_keys import KeyRing -from server.plugins.plugin_base import finishing_task +from server.entities.plugin_base import finishing_task APP_ID = KeyRing().get("tacyt-appid") SECRET_KEY = KeyRing().get("tacyt-secret") diff --git a/server/plugins/threatcrowd.py b/server/plugins/threatcrowd.py index 317fc77..8e42743 100755 --- a/server/plugins/threatcrowd.py +++ b/server/plugins/threatcrowd.py @@ -2,9 +2,9 @@ import json import requests -from server.entities.resource import Resources, ResourceType +from server.entities.resource_types import ResourceType from tasks.tasks import celery_app -from server.plugins.plugin_base import finishing_task +from server.entities.plugin_base import finishing_task URL_IP = "https://www.threatcrowd.org/searchApi/v2/ip/report/?ip={ip}" URL_DOMAIN = "https://www.threatcrowd.org/searchApi/v2/domain/report/?domain={domain}" diff --git a/server/plugins/threatminer.py b/server/plugins/threatminer.py index bdbd639..f0c0477 100755 --- a/server/plugins/threatminer.py +++ b/server/plugins/threatminer.py @@ -2,7 +2,8 @@ import json import traceback -from server.entities.resource import Resources, ResourceType +from server.entities.resource_types import ResourceType +from server.entities.resource_manager import ResourceManager from tasks.tasks import celery_app import json import requests @@ -210,7 +211,7 @@ def threatminer_task(plugin_name, project_id, resource_id, resource_type, domain else: print("threatminer resource type does not found") - resource = Resources.get(resource_id, resource_type) + resource = ResourceManager.get(resource_id, resource_type) resource.set_plugin_results( plugin_name, project_id, resource_id, resource_type, query_result ) diff --git a/server/plugins/urlscan.py b/server/plugins/urlscan.py index 5ecc2f7..478da97 100755 --- a/server/plugins/urlscan.py +++ b/server/plugins/urlscan.py @@ -3,10 +3,10 @@ import time import requests -from server.entities.resource import Resources, ResourceType +from server.entities.resource_types import ResourceType from tasks.tasks import celery_app from tasks.api_keys import KeyRing -from server.plugins.plugin_base import finishing_task +from server.entities.plugin_base import finishing_task API_KEY = KeyRing().get("urlscan") SUBMISSION_URL = "https://urlscan.io/api/v1/scan/" diff --git a/server/plugins/verifymail.py b/server/plugins/verifymail.py index 8038825..1b154e9 100755 --- a/server/plugins/verifymail.py +++ b/server/plugins/verifymail.py @@ -3,9 +3,9 @@ import requests from tasks.api_keys import KeyRing -from server.entities.resource import Resources, ResourceType +from server.entities.resource_types import ResourceType from tasks.tasks import celery_app -from server.plugins.plugin_base import finishing_task +from server.entities.plugin_base import finishing_task API_KEY = KeyRing().get("verify-email") URL = "https://app.verify-email.org/api/v1/{key}/verify/{email}" diff --git a/server/plugins/virustotal.py b/server/plugins/virustotal.py index e5012a2..d1a3634 100755 --- a/server/plugins/virustotal.py +++ b/server/plugins/virustotal.py @@ -4,8 +4,8 @@ from tasks.api_keys import KeyRing from tasks.tasks import celery_app -from server.entities.resource import Resources, ResourceType -from server.plugins.plugin_base import finishing_task +from server.entities.resource_types import ResourceType +from server.entities.plugin_base import finishing_task API_KEY = KeyRing().get("virustotal") diff --git a/server/plugins/vt_domain.py b/server/plugins/vt_domain.py index 06e04bd..63b9ed3 100644 --- a/server/plugins/vt_domain.py +++ b/server/plugins/vt_domain.py @@ -4,8 +4,8 @@ from tasks.api_keys import KeyRing from tasks.tasks import celery_app -from server.entities.resource import Resources, ResourceType -from server.plugins.plugin_base import finishing_task +from server.entities.resource_types import ResourceType +from server.entities.plugin_base import finishing_task API_KEY = KeyRing().get("virustotal") diff --git a/server/plugins/whois.py b/server/plugins/whois.py index 5acec0f..394765d 100755 --- a/server/plugins/whois.py +++ b/server/plugins/whois.py @@ -4,9 +4,9 @@ import json import traceback -from server.entities.resource import Resources, ResourceType +from server.entities.resource_types import ResourceType from tasks.tasks import celery_app -from server.plugins.plugin_base import finishing_task +from server.entities.plugin_base import finishing_task # Which resources are this plugin able to work with RESOURCE_TARGET = [ResourceType.DOMAIN, ResourceType.EMAIL] diff --git a/server/routes/apikeys.py b/server/routes/apikeys.py index fa47ed3..a6fde69 100755 --- a/server/routes/apikeys.py +++ b/server/routes/apikeys.py @@ -8,7 +8,6 @@ from server.db import DB from server.utils.password import token_required -from server.entities.user import User apikeys_api = Blueprint("apikeys", __name__) @@ -17,7 +16,7 @@ @token_required def get_apikeys(user): try: - results = DB("apikeys").collection.find({}, {'_id': False}) + results = DB("apikeys").collection.find({}, {"_id": False}) list_results = list(results) return dumps(list_results) @@ -26,6 +25,7 @@ def get_apikeys(user): print(e) return jsonify({"error_message": "Error getting API keys"}), 400 + @apikeys_api.route("/api/upload_apikeys", methods=["POST"]) @token_required def upload_apikeys(user): @@ -42,6 +42,7 @@ def upload_apikeys(user): print(e) return jsonify({"error_message": "Error uploading API keys"}), 400 + @apikeys_api.route("/api/remove_apikeys", methods=["POST"]) @token_required def remove_apikeys(user): @@ -49,9 +50,7 @@ def remove_apikeys(user): apikeys = request.json["entries"] for name in apikeys: print(name) - result = DB("apikeys").collection.remove( - {"name": name["name"]} - ) + result = DB("apikeys").collection.remove({"name": name["name"]}) return json.dumps(apikeys, default=str) diff --git a/server/routes/plugins.py b/server/routes/plugins.py index 98e2b0f..55524bd 100755 --- a/server/routes/plugins.py +++ b/server/routes/plugins.py @@ -10,7 +10,7 @@ from server.utils.password import token_required -from server.entities.resource import Resources +from server.entities.resource_manager import ResourceManager from server.entities.resource_types import ResourceType, ResourceTypeException from server.entities.user import User from server.entities.pastebin_manager import PastebinManager @@ -38,7 +38,7 @@ def get_all_plugins(user): return json.dumps(plugins) except Exception as e: - print(e) + print(f"[get_all_plugins]: {e}") return jsonify({"error_message": "Error gettings plugins"}), 400 @@ -52,7 +52,7 @@ def get_plugins(user): project = User(user).get_active_project() resource_type = ResourceType(resource_type_as_string) - resource = Resources.get(resource_id, resource_type) + resource = ResourceManager.get(resource_id) plugin_list = resource.get_plugins(project_id) return json.dumps(plugin_list, default=str) @@ -72,7 +72,7 @@ def launch_plugin(user): project = User(user).get_active_project() resource_type = ResourceType(resource_type_as_string) - resource = Resources.get(resource_id, resource_type) + resource = ResourceManager.get(resource_id) resource.launch_plugin(project.get_id(), plugin_name) return jsonify({"sucess_message": "ok"}) diff --git a/server/routes/resources.py b/server/routes/resources.py index 31c77bf..e596751 100755 --- a/server/routes/resources.py +++ b/server/routes/resources.py @@ -9,73 +9,15 @@ from server.utils.password import token_required -from server.entities.resource import Resources +from server.entities.resource_manager import ResourceManager + from server.entities.resource_types import ResourceType, ResourceTypeException from server.entities.user import User +from server.entities.project import Project resources_api = Blueprint("resources", __name__) -@resources_api.route("/api/create_multiple_resources", methods=["POST"]) -@token_required -def create_multiple_resources(user): - try: - project = User(user).get_active_project() - response = [] - - for entry in request.json["entries"]: - resource_name = entry["resource"] - resource_type = ResourceType.get_type_from_string(entry["type"].lower()) - if resource_type == ResourceType.UNKNOWN: - print( - f"[resources/create_multiple_resources]: Unknown resource type {entry['type'].lower()}" - ) - continue - - resource = Resources.get(resource_name, resource_type, get_by_name=True) - project.add_resource(resource) - - response.append( - { - "success_message": f"Added new resource: {resource_name}", - "new_resource": resource.to_JSON(), - "type": resource.get_type_value(), - } - ) - resource.launch_plugins(project.get_id()) - - # Deal with the case of URL resources where we have the chance to add a Domain or IP - if resource.get_type() == ResourceType.URL: - ip_or_domain = urllib.parse.urlparse(resource_name).netloc - resource_type = ResourceType.validate_ip_or_domain(ip_or_domain) - if ip_or_domain: - resource = Resources.get( - ip_or_domain, resource_type, get_by_name=True - ) - project.add_resource(resource) - response.append( - { - "success_message": f"Added new resource: {ip_or_domain}", - "new_resource": resource.to_JSON(), - "type": resource.get_type_value(), - } - ) - resource.launch_plugins(project.get_id()) - - # TODO: Deal with the case of domain -> IP - # TODO: Deal with the case of emails -> domains -> IP - - return jsonify(response) - - except ResourceTypeException: - return jsonify({"error_message": "Trying to add an unknown resource type"}), 400 - - except Exception as e: - tb1 = traceback.TracebackException.from_exception(e) - print("".join(tb1.format())) - return jsonify({"error_message": "Server error :("}), 400 - - @resources_api.route("/api/create_resource", methods=["POST"]) @token_required def create_resource(user): @@ -85,28 +27,25 @@ def create_resource(user): resource_type = ResourceType.get_type_from_string(resource_type) - resource = Resources.get(resource_name, resource_type, get_by_name=True) + resource, created = ResourceManager.get_or_create(resource_name, resource_type) project = User(user).get_active_project() project.add_resource(resource) response = [] - response.append( - { - "success_message": f"Added new resource: {resource_name}", - "new_resource": resource.to_JSON(), - "type": resource.get_type_value(), - } - ) + response.append(resource.to_JSON()) - resource.launch_plugins(project.get_id()) + if created: + resource.launch_plugins(project.get_id()) # Deal with the case of URL resources where we have the chance to add a Domain or IP if resource.get_type() == ResourceType.URL: ip_or_domain = urllib.parse.urlparse(resource_name).netloc resource_type = ResourceType.validate_ip_or_domain(ip_or_domain) if ip_or_domain: - resource = Resources.get(ip_or_domain, resource_type, get_by_name=True) + resource, created = ResourceManager.get_or_create( + ip_or_domain, resource_type + ) project.add_resource(resource) response.append( { @@ -115,7 +54,8 @@ def create_resource(user): "type": resource.get_type_value(), } ) - resource.launch_plugins(project.get_id()) + if created: + resource.launch_plugins(project.get_id()) # TODO: Deal with the case of domain -> IP # TODO: Deal with the case of emails -> domains -> IP @@ -134,26 +74,30 @@ def create_resource(user): @resources_api.route("/api/get_resources", methods=["POST"]) @token_required def get_resources(user): - resource_type_as_string = request.json["type"] - try: - resource_type = ResourceType(resource_type_as_string) + project_id = request.json["project_id"] + user_projects = [str(project) for project in User(user).get_projects()] - project = User(user).get_active_project() - resources = project.get_resources(resource_type) + # User unable to load the project + if not project_id in user_projects: + return ( + jsonify( + { + "error_message": f"User is not allowed to load project {project_id}" + } + ), + 400, + ) + + project = Project(project_id) + resources = project.get_resources() results = [] for resource in resources: - results.append(Resources.get(resource, resource_type).to_JSON()) + results.append(ResourceManager.get(resource).to_JSON()) return jsonify(results) - except ValueError: - raise ResourceTypeException() - - except ResourceTypeException: - return jsonify({"error_message": "Received an unknown type of resource"}), 400 - except Exception as e: print(f"Error getting resource list {e}") tb1 = traceback.TracebackException.from_exception(e) @@ -164,6 +108,9 @@ def get_resources(user): @resources_api.route("/api/unlink_resource", methods=["POST"]) @token_required def unlink_resource(user): + """ + Unlink (not remove) a resource from the active project. + """ try: resource_id = bson.ObjectId(request.json["resource_id"]) @@ -181,15 +128,17 @@ def unlink_resource(user): @token_required def get_resource(user): """ - Return a resource doc + Return a resource json dump by its ID """ - resource_type_as_string = request.json["resource_type"] + resource_id = request.json["resource_id"] try: - resource_type = ResourceType(resource_type_as_string) - resource = Resources.get(resource_id, resource_type) - return jsonify(resource.to_JSON()) + resource = ResourceManager.get(resource_id) + if resource: + return jsonify(resource.to_JSON()) + else: + return jsonify({"error_message": "Resource not found"}) except ValueError: raise ResourceTypeException() diff --git a/server/routes/tags.py b/server/routes/tags.py index c104da4..23bf6d7 100755 --- a/server/routes/tags.py +++ b/server/routes/tags.py @@ -9,7 +9,7 @@ from server.utils.password import token_required -from server.entities.resource import Resources +from server.entities.resource_manager import ResourceManager from server.entities.resource_types import ResourceType, ResourceTypeException from server.entities.user import User @@ -83,7 +83,7 @@ def tag_to_resource(user): tag = request.json["tag"] resource_type = ResourceType(resource_type_as_string) - resource = Resources.get(resource_id, resource_type) + resource = ResourceManager.get(resource_id) resource.manage_tag(tag) return jsonify({"sucess_message": "ok"}) diff --git a/tasks/subtasks/googlesearch.py b/tasks/googlesearch.py old mode 100755 new mode 100644 similarity index 99% rename from tasks/subtasks/googlesearch.py rename to tasks/googlesearch.py index 334d91e..418722e --- a/tasks/subtasks/googlesearch.py +++ b/tasks/googlesearch.py @@ -3,11 +3,8 @@ import requests from googleapiclient.discovery import build - - from tasks.api_keys import KeyRing - API_KEY = KeyRing().get("googlesearch") SORT = "date" diff --git a/tasks/subtasks/__init__.py b/tasks/subtasks/__init__.py deleted file mode 100755 index e69de29..0000000 diff --git a/tasks/subtasks/abuseipdb.py b/tasks/subtasks/abuseipdb.py deleted file mode 100755 index 9998b95..0000000 --- a/tasks/subtasks/abuseipdb.py +++ /dev/null @@ -1,33 +0,0 @@ -import traceback -import json -import requests - -from tasks.api_keys import KeyRing - -API_KEY = KeyRing().get("abuseipdb") -URL = "https://api.abuseipdb.com/api/v2/check" - - -def abuseipdb(ip): - try: - if not API_KEY: - print("No API key...!") - return None - - response = {} - headers = {"Accept": "application/json", "Key": API_KEY} - data = {"ipAddress": ip} - abuse_response = requests.get(URL, headers=headers, json=data) - if not abuse_response.status_code == 200: - print("API key error!") - return None - else: - response = json.loads(abuse_response.content) - print(response) - - return response - - except Exception as e: - tb1 = traceback.TracebackException.from_exception(e) - print("".join(tb1.format())) - return None diff --git a/tasks/subtasks/asn.py b/tasks/subtasks/asn.py deleted file mode 100755 index a6bc59f..0000000 --- a/tasks/subtasks/asn.py +++ /dev/null @@ -1,61 +0,0 @@ -def asn(ip): - try: - results = {} - target = IPWhois(ip) - lookup = target.lookup_rdap(depth=1) - if lookup: - results["asn"] = { - "asn": lookup["asn"], - "asn_cidr": lookup["asn_cidr"], - "asn_country_code": lookup["asn_country_code"], - "asn_date": lookup["asn_date"], - "asn_description": lookup["asn_description"], - "asn_registry": lookup["asn_registry"], - } - - results["network"] = { - "cidr": lookup["network"]["cidr"], - "country": lookup["network"]["country"], - "handle": lookup["network"]["handle"], - "name": lookup["network"]["name"], - } - - return results - - except Exception as e: - tb1 = traceback.TracebackException.from_exception(e) - print("".join(tb1.format())) - -@celery_app.task -def basic_ip_task(plugin_name, project_id, resource_id, resource_type, ip): - - query_result = {} - - # PTR - try: - PTR_record = ptr(ip) - - if PTR_record: - query_result["ptr"] = PTR_record - - ASN_NET_record = asn(ip) - - if "asn" in ASN_NET_record: - query_result["asn"] = ASN_NET_record["asn"] - - if "network" in ASN_NET_record: - query_result["network"] = ASN_NET_record["network"] - - # TODO: Probably, we can save some parameters here when object is instantiated - resource_type = ResourceType(resource_type) - - resource = Resources.get(resource_id, resource_type) - resource.set_plugin_results( - plugin_name, project_id, resource_id, resource_type, query_result - ) - - except Exception as e: - tb1 = traceback.TracebackException.from_exception(e) - print("".join(tb1.format())) - - diff --git a/tasks/subtasks/binaryedge.py b/tasks/subtasks/binaryedge.py deleted file mode 100755 index 502527d..0000000 --- a/tasks/subtasks/binaryedge.py +++ /dev/null @@ -1,26 +0,0 @@ -def binaryedge(ip): - try: - - if not API_KEY: - print("No API key...!") - return {} - - response = {} - headers = {"X-Key": API_KEY} - response = requests.get(URL.format(**{"ip": ip}), headers=headers) - - if response.status_code == 404: - print("associated records not found!") - elif not response.status_code == 200: - print("API key error!") - else: - response = json.loads(response.content) - print(response) - return response - - return {} - - except Exception as e: - tb1 = traceback.TracebackException.from_exception(e) - print("".join(tb1.format())) - return None diff --git a/tasks/subtasks/botscout.py b/tasks/subtasks/botscout.py deleted file mode 100644 index 2221309..0000000 --- a/tasks/subtasks/botscout.py +++ /dev/null @@ -1,49 +0,0 @@ -# Login with the following information: -# Login Name: zigineki@getnada.com -# Password: 6aiIRgAm -# The BotScout Team -# http://BotScout.com - - -import traceback -import json -import urllib.request -from bs4 import BeautifulSoup - -from tasks.api_keys import KeyRing - -API_KEY = KeyRing().get("botscout") - - -def botscout_ip(ip): - try: - URL = f"http://botscout.com/test/?ip={ip}&key={API_KEY}&format=xml" - response = urllib.request.urlopen(URL).read() - return response - - except Exception as e: - tb1 = traceback.TracebackException.from_exception(e) - print("".join(tb1.format())) - return None - - -# parsing results -def botscout_ip_details(ip): - try: - URL = f"http://botscout.com/search.htm?sterm={ip}&stype=q" - response = urllib.request.urlopen(URL).read() - - # parser HTML to extract last or To 10 - # soup = BeautifulSoup(response, 'html.parser') - # html_data = soup.find('table', attrs={'class':'sortable'}) - # table_data = [[cell.text for cell in row("td")]] - # for row in soup.body.find_all('table', attrs={'class' : 'sortable'})] - - # print(json.dumps(dict(table_data))) - - return response - - except Exception as e: - tb1 = traceback.TracebackException.from_exception(e) - print("".join(tb1.format())) - return None diff --git a/tasks/subtasks/cybercrimetracker.py b/tasks/subtasks/cybercrimetracker.py deleted file mode 100755 index 48e09b9..0000000 --- a/tasks/subtasks/cybercrimetracker.py +++ /dev/null @@ -1,35 +0,0 @@ -import traceback -import json -import requests, base64 -from bs4 import BeautifulSoup - -#from tasks.api_keys import KeyRing - -#API_KEY = KeyRing().get("hybrid") - -URL = "http://cybercrime-tracker.net/index.php?search={ip}" - -def cybercrime_search(ip): - try: - - response = {} - - headers = { "User-Agent": "Mozilla/5.0 (X11; Linux i686; rv:64.0) Gecko/20100101 Firefox/64.0" } - - response = requests.get(URL.format(**{"ip": ip}), headers=headers) - if not response.status_code == 200: - print("Response Service error!") - return None - else: - soup = BeautifulSoup(response.content, 'html.parser') - #response = json.loads(response.content) - - - return soup - - except Exception as e: - tb1 = traceback.TracebackException.from_exception(e) - print("".join(tb1.format())) - return None - -#print ( cybercrime_search("5.79.66.145") ) diff --git a/tasks/subtasks/diario.py b/tasks/subtasks/diario.py deleted file mode 100755 index e69de29..0000000 diff --git a/tasks/subtasks/dns.py b/tasks/subtasks/dns.py deleted file mode 100755 index bca4858..0000000 --- a/tasks/subtasks/dns.py +++ /dev/null @@ -1,60 +0,0 @@ -import dns.resolver as resolver -import traceback - -from urllib.parse import urlparse -from server.entities.resource import Resources, ResourceType - -from tasks.tasks import celery_app - -# If you want to expand DNS query types this is the right place -LOOKUP = ["NS", "A", "AAAA", "MX", "TXT", "SRV"] - - -def dns(domain): - try: - results = {} - - for TYPE in LOOKUP: - try: - r = resolver.query(domain, TYPE) - results[TYPE] = [str(i) for i in r] - - except: - # Case when the query must be on canonical domain - try: - root_name = ".".join(domain.split(".")[-2:]) - r = resolver.query(root_name, TYPE) - results[TYPE] = [str(i) for i in r] - - except: - results[TYPE] = None - - print(results) - return results - - except Exception as e: - tb1 = traceback.TracebackException.from_exception(e) - print("".join(tb1.format())) - -@celery_app.task -def dns_task(plugin_name, project_id, resource_id, resource_type, domain): - - query_result = {} - - # PTR - try: - dns_results = dns(domain) - query_result = dns_results - - # TODO: Probably, we can save some parameters here when object is instantiated - resource_type = ResourceType(resource_type) - - resource = Resources.get(resource_id, resource_type) - resource.set_plugin_results( - plugin_name, project_id, resource_id, resource_type, query_result - ) - - except Exception as e: - tb1 = traceback.TracebackException.from_exception(e) - print("".join(tb1.format())) - diff --git a/tasks/subtasks/emailrep.py b/tasks/subtasks/emailrep.py deleted file mode 100755 index 139597f..0000000 --- a/tasks/subtasks/emailrep.py +++ /dev/null @@ -1,2 +0,0 @@ - - diff --git a/tasks/subtasks/geoip.py b/tasks/subtasks/geoip.py deleted file mode 100755 index 9bd26d6..0000000 --- a/tasks/subtasks/geoip.py +++ /dev/null @@ -1,19 +0,0 @@ -import traceback -import json -import urllib.request - -from tasks.api_keys import KeyRing - -API_KEY = KeyRing().get("ipstack") - - -def geoip(ip): - try: - URL = f"http://api.ipstack.com/{ip}?access_key={API_KEY}&format=1" - response = urllib.request.urlopen(URL).read() - return json.loads(response) - - except Exception as e: - tb1 = traceback.TracebackException.from_exception(e) - print("".join(tb1.format())) - return None diff --git a/tasks/subtasks/haveibeenpwned.py b/tasks/subtasks/haveibeenpwned.py deleted file mode 100755 index e69de29..0000000 diff --git a/tasks/subtasks/hunterio.py b/tasks/subtasks/hunterio.py deleted file mode 100755 index e69de29..0000000 diff --git a/tasks/subtasks/hybridanalysis.py b/tasks/subtasks/hybridanalysis.py deleted file mode 100644 index 5c64103..0000000 --- a/tasks/subtasks/hybridanalysis.py +++ /dev/null @@ -1,71 +0,0 @@ -# https://www.hybrid-analysis.com/ -# user: zigineki@getnada.com -# pass: Developer1 - -import traceback -import json -import requests, base64 - -from tasks.api_keys import KeyRing - -API_KEY = KeyRing().get("hybrid") -SECRET = KeyRing().get("hybrid_secret") - -URL = "https://www.hybrid-analysis.com/api/v2" -PATH_HASH = "/search/hash" -PATH_TERMS = "/search/terms" - - -def hybrid_search_hash(hash): - try: - if not API_KEY: - print("No API key...!") - return None - - response = {} - - headers = {"User-Agent": "Falcon Sandbox", "api-key": API_KEY} - post_data = {"hash": hash} - - response = requests.post(URL + PATH_HASH, post_data, headers=headers) - if not response.status_code == 200: - print("API key error!") - return None - else: - response = json.loads(response.content) - - return response - - except Exception as e: - tb1 = traceback.TracebackException.from_exception(e) - print("".join(tb1.format())) - return None - - -# https://www.hybrid-analysis.com/docs/api/v2#/Search/post_search_terms -# param = host, domain, url -def hybrid_search_terms(param): - try: - if not API_KEY: - print("No API key...!") - return None - - response = {} - - headers = {"User-Agent": "Falcon Sandbox", "api-key": API_KEY} - - post_data = {"domain": param} - - response = requests.post(URL + PATH_TERMS, post_data, headers=headers) - if not response.status_code == 200: - print("API key error!") - return None - else: - response = json.loads(response.content) - - return response - - except Exception as e: - tb1 = traceback.TracebackException.from_exception(e) - print("".join(tb1.format())) - return None diff --git a/tasks/subtasks/maltiverse.py b/tasks/subtasks/maltiverse.py deleted file mode 100755 index e69de29..0000000 diff --git a/tasks/subtasks/onyphe.py b/tasks/subtasks/onyphe.py deleted file mode 100755 index e69de29..0000000 diff --git a/tasks/subtasks/otx.py b/tasks/subtasks/otx.py deleted file mode 100644 index e69de29..0000000 diff --git a/tasks/subtasks/pastebin.py b/tasks/subtasks/pastebin.py deleted file mode 100755 index e062c94..0000000 --- a/tasks/subtasks/pastebin.py +++ /dev/null @@ -1,67 +0,0 @@ - -def get_key_from_paste_key(item): - return item.split("/")[-1] - - -def pastebin(results): - try: - if not API_KEY: - raise Exception("No API_KEY for pastebin") - - links = [] - if "items" in results: - links = [item["link"] for item in results["items"]] - pastebins_refs = [] - - for link in links: - paste_key = get_key_from_paste_key(link) - try: - time.sleep(RATE_LIMIT) - - args = {} - meta = requests.get(METADATA_URL + paste_key) - if meta.status_code == 200: - try: - meta = meta.json()[0] - except: - print(f"This paste {paste_key} has been deleted") - continue - else: - print(f"Error {meta.status_code} getting paste from pastebin.com") - continue - - for field in [ - "size", - "title", - "user", - "hits", - "date", - "syntax", - "expire", - ]: - args[field] = meta[field] - - paste = Paste(paste_key, args) - - time.sleep(RATE_LIMIT) - - result = requests.get(RAWPASTE_URL + paste_key) - - if result.status_code == 200: - paste.set_content(result.content) - - pastebins_refs.append(paste.save()) - - except Exception as e: - print(f"Failed to get pastebin: {link}") - tb1 = traceback.TracebackException.from_exception(e) - print("".join(tb1.format())) - continue - - if len(pastebins_refs) == 0 or len(links) == 0: - return None - return pastebins_refs - - except Exception as e: - tb1 = traceback.TracebackException.from_exception(e) - print("".join(tb1.format())) diff --git a/tasks/subtasks/phishtank.py b/tasks/subtasks/phishtank.py deleted file mode 100755 index e69de29..0000000 diff --git a/tasks/subtasks/ptr.py b/tasks/subtasks/ptr.py deleted file mode 100755 index 139597f..0000000 --- a/tasks/subtasks/ptr.py +++ /dev/null @@ -1,2 +0,0 @@ - - diff --git a/tasks/subtasks/pulsedive.py b/tasks/subtasks/pulsedive.py deleted file mode 100644 index e69de29..0000000 diff --git a/tasks/subtasks/robtex.py b/tasks/subtasks/robtex.py deleted file mode 100755 index b28b04f..0000000 --- a/tasks/subtasks/robtex.py +++ /dev/null @@ -1,3 +0,0 @@ - - - diff --git a/tasks/subtasks/sherlock.py b/tasks/subtasks/sherlock.py deleted file mode 100755 index e69de29..0000000 diff --git a/tasks/subtasks/shodan.py b/tasks/subtasks/shodan.py deleted file mode 100755 index 082ca43..0000000 --- a/tasks/subtasks/shodan.py +++ /dev/null @@ -1,53 +0,0 @@ -import traceback -import json -import requests - - -from tasks.api_keys import KeyRing - -API_KEY = KeyRing().get("shodan") - -URL = "https://api.shodan.io/shodan/host/{ip}?key={API_KEY}" - - -def shodan(ip): - try: - if not API_KEY: - print("No API key...!") - return None - - response = {} - ipinfo = requests.get(URL.format(**{"ip": ip, "API_KEY": API_KEY})) - if not ipinfo.status_code == 200: - return None - else: - ipinfo = json.loads(ipinfo.content) - with open("/tmp/shodan.json", "w") as f: - f.write(json.dumps(ipinfo)) - - response["hostnames"] = ipinfo["hostnames"] if "hostnames" in ipinfo else [] - response["os"] = ipinfo["os"] if "os" in ipinfo else None - response["org"] = ipinfo["org"] if "org" in ipinfo else None - response["isp"] = ipinfo["isp"] if "isp" in ipinfo else None - response["services"] = [] - - for data in ipinfo["data"]: - service = {} - service["port"] = data["port"] if "port" in data else None - service["transport"] = data["transport"] if "transport" in data else None - service["service"] = data["service"] if "service" in data else None - service["data"] = data["data"] if "data" in data else None - service["product"] = data["product"] if "product" in data else None - service["banner"] = data["banner"] if "banner" in data else None - service["devicetype"] = data["devicetype"] if "devicetype" in data else None - service["timestamp"] = data["timestamp"] if "timestamp" in data else None - service["hostnames"] = data["hostnames"] if "hostnames" in data else [] - - response["services"].append(service) - - return response - - except Exception as e: - tb1 = traceback.TracebackException.from_exception(e) - print("".join(tb1.format())) - return None diff --git a/tasks/subtasks/tacyt.py b/tasks/subtasks/tacyt.py deleted file mode 100755 index 8b13789..0000000 --- a/tasks/subtasks/tacyt.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/tasks/subtasks/threatcrowd.py b/tasks/subtasks/threatcrowd.py deleted file mode 100755 index e69de29..0000000 diff --git a/tasks/subtasks/threatminer.py b/tasks/subtasks/threatminer.py deleted file mode 100644 index 8b13789..0000000 --- a/tasks/subtasks/threatminer.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/tasks/subtasks/urlscan.py b/tasks/subtasks/urlscan.py deleted file mode 100755 index 8b13789..0000000 --- a/tasks/subtasks/urlscan.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/tasks/subtasks/verifymail.py b/tasks/subtasks/verifymail.py deleted file mode 100755 index e69de29..0000000 diff --git a/tasks/subtasks/virustotal.py b/tasks/subtasks/virustotal.py deleted file mode 100755 index 5d9eb7d..0000000 --- a/tasks/subtasks/virustotal.py +++ /dev/null @@ -1,56 +0,0 @@ -import traceback -import json -import requests - -from tasks.api_keys import KeyRing -from server.entities.resource_types import ResourceType - -API_KEY = KeyRing().get("virustotal") - -url_for_hashes = "https://www.virustotal.com/vtapi/v2/file/report" -url_for_urls = "https://www.virustotal.com/vtapi/v2/url/report" -url_for_domains = "https://www.virustotal.com/vtapi/v2/domain/report" -url_for_ips = "https://www.virustotal.com/vtapi/v2/ip-address/report" - - -def virustotal(resource, resource_type): - try: - if not API_KEY: - print("No API key...!") - return None - - response = None - url = None - params = {"apikey": API_KEY} - - if resource_type == ResourceType.DOMAIN: - url = url_for_domains - params["domain"] = resource - elif resource_type == ResourceType.URL: - url = url_for_urls - params["resource"] = resource - elif resource_type == ResourceType.IPv4: - url = url_for_ips - params["ip"] = resource - elif resource_type == ResourceType.HASH: - url = url_for_hashes - params["resource"] = resource - else: - print("[VT] Unknown resource type before querying service") - return None - - response = requests.get(url, params=params) - - if not response.status_code == 200: - print(response) - return None - else: - response = json.loads(response.content) - - print(response) - return response - - except Exception as e: - tb1 = traceback.TracebackException.from_exception(e) - print("".join(tb1.format())) - return None diff --git a/tasks/tasks.py b/tasks/tasks.py index 1ef2a98..4af9964 100755 --- a/tasks/tasks.py +++ b/tasks/tasks.py @@ -3,7 +3,7 @@ from celery import Celery # TODO: Integrar en una sola funcion, se repeite en plugins.py -EXCLUDE_SET = ["plugins.py", "__init__.py", "plugin_base.py", "TEMPLATE.py"] +EXCLUDE_SET = ["__init__.py", "TEMPLATE.py"] plugins = [] for root, dirs, files in os.walk("server/plugins"):