Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feat/dns loc records #3349

Open
wants to merge 40 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
40 commits
Select commit Hold shift + click to select a range
543ba75
Work in progress maxmind geoip boefje
dekkers Jul 15, 2024
8fead2a
add katalogus settings, fix reader
noamblitz Jul 15, 2024
cc447d3
add geographic point ooi
noamblitz Jul 15, 2024
86d103e
add geo normalizer
noamblitz Jul 15, 2024
097c67e
fix caching
noamblitz Jul 15, 2024
9a973a6
add schema
noamblitz Jul 15, 2024
3494a3c
add cover
noamblitz Jul 15, 2024
476b1a8
update schema
noamblitz Jul 16, 2024
c391a09
styling
noamblitz Jul 16, 2024
c796705
timezone awareness
noamblitz Jul 16, 2024
068c8a8
update tests
noamblitz Jul 16, 2024
052178b
update tests
noamblitz Jul 16, 2024
c7184d6
Update boefjes/boefjes/plugins/kat_maxmind_geoip/normalizer.json
noamblitz Jul 18, 2024
0c489ce
Update boefjes/boefjes/plugins/kat_maxmind_geoip/main.py
noamblitz Jul 26, 2024
e15ba57
Update octopoes/octopoes/models/ooi/geography.py
noamblitz Jul 26, 2024
c827e5d
change consumes of normalizer
noamblitz Jul 26, 2024
b32f1c4
Update octopoes/octopoes/models/ooi/geography.py
noamblitz Jul 26, 2024
17531fd
compress cover
noamblitz Jul 26, 2024
6e2c0da
better type hinting
noamblitz Jul 26, 2024
d136149
flexible date in dir name
noamblitz Jul 29, 2024
fbdd0ad
Update normalize.py
underdarknl Jul 30, 2024
03da314
Update main.py
underdarknl Jul 30, 2024
b625527
Update records.py
underdarknl Jul 30, 2024
0108e17
Update normalize.py
underdarknl Jul 31, 2024
6cf7ba7
Update main.py
underdarknl Jul 31, 2024
5302bf9
add GPOS records to normalize.py
underdarknl Jul 31, 2024
a9e5c52
Update records.py
underdarknl Jul 31, 2024
d8bfae2
Merge branch 'main' into feat/dns-loc-records
noamblitz Aug 12, 2024
566f958
inherit from base dns
noamblitz Aug 12, 2024
858711f
better check
noamblitz Aug 12, 2024
cf3c911
linting
noamblitz Aug 12, 2024
e7db12e
linting
noamblitz Aug 12, 2024
987d2e7
add loc records
noamblitz Aug 12, 2024
7aace87
fix import
noamblitz Aug 12, 2024
f084b35
Merge branch 'main' into feat/dns-loc-records
underdarknl Aug 15, 2024
4ec7896
Merge branch 'main' into feat/dns-loc-records
noamblitz Aug 19, 2024
b76963b
inherit from dnslocation base
noamblitz Aug 20, 2024
e3cd9a3
type hint float
noamblitz Aug 20, 2024
92a9d8d
better naming
noamblitz Aug 20, 2024
bcff162
Update octopoes/octopoes/models/ooi/dns/records.py
ammar92 Aug 20, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions boefjes/boefjes/plugins/kat_dns/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@
"CNAME",
"DNAME",
"SOA",
"LOC",
"GPOS",
}


Expand Down
34 changes: 34 additions & 0 deletions boefjes/boefjes/plugins/kat_dns/normalize.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@
from dns.message import Message, from_text
from dns.rdtypes.ANY.CAA import CAA
from dns.rdtypes.ANY.CNAME import CNAME
from dns.rdtypes.ANY.GPOS import GPOS
from dns.rdtypes.ANY.LOC import LOC
from dns.rdtypes.ANY.MX import MX
from dns.rdtypes.ANY.NS import NS
from dns.rdtypes.ANY.SOA import SOA
Expand All @@ -21,6 +23,8 @@
DNSARecord,
DNSCAARecord,
DNSCNAMERecord,
DNSGPOSRecord,
DNSLOCRecord,
DNSMXRecord,
DNSNSRecord,
DNSRecord,
Expand All @@ -29,6 +33,7 @@
)
from octopoes.models.ooi.dns.zone import DNSZone, Hostname
from octopoes.models.ooi.email_security import DKIMExists, DMARCTXTRecord
from octopoes.models.ooi.geography import GeographicPoint
from octopoes.models.ooi.network import IPAddressV4, IPAddressV6, Network


Expand Down Expand Up @@ -159,6 +164,35 @@ def register_record(record: DNSRecord) -> DNSRecord:
default_args["value"] = record_value[2]
register_record(DNSCAARecord(**default_args))

if isinstance(rr, LOC | GPOS):
default_args.update(
{
"value": rr.to_text(),
"latitude": rr.float_lontitude(),
"longitude": rr.float_longtitude(),
"altitude": rr.float_altitude(),
}
)

if isinstance(rr, LOC):
default_args.update(
{
"horizontal_precision": rr.horizontal_precision(),
"vertical_precision": rr.vertical_precision(),
"size": rr.size(),
}
)
location_record = register_record(DNSLOCRecord(**default_args))
else:
location_record = register_record(DNSGPOSRecord(**default_args))

geo_point = {
"ooi": location_record.reference,
"latitude": rr.float_lontitude(),
"longitude": rr.float_longtitude(),
}
register_record(GeographicPoint(**geo_point))

# link the hostnames to their discovered zones
for hostname_, zone in zone_links.items():
hostname_store[hostname_].dns_zone = zone.reference
Expand Down
1,422 changes: 600 additions & 822 deletions boefjes/poetry.lock

Large diffs are not rendered by default.

1,309 changes: 565 additions & 744 deletions boefjes/requirements-dev.txt

Large diffs are not rendered by default.

1,303 changes: 562 additions & 741 deletions boefjes/requirements.txt

Large diffs are not rendered by default.

34 changes: 33 additions & 1 deletion octopoes/octopoes/models/ooi/dns/records.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@

class DNSRecord(OOI):
hostname: Reference = ReferenceField(Hostname, max_issue_scan_level=0, max_inherit_scan_level=2)
dns_record_type: Literal["A", "AAAA", "CAA", "CNAME", "MX", "NS", "PTR", "SOA", "SRV", "TXT"]
dns_record_type: Literal["A", "AAAA", "CAA", "CNAME", "MX", "NS", "PTR", "SOA", "SRV", "TXT", "LOC", "GPOS"]
value: str
ttl: int | None = None # todo: validation

Expand Down Expand Up @@ -185,3 +185,35 @@ class DNSCAARecord(DNSRecord):
# without interior spaces or (2) a quoted string.
value: str
_natural_key_attrs = ["hostname", "flags", "tag", "value"]


class DNSLocation(DNSRecord):
latitude: float | None = None
longitude: float | None = None
altitude: float | None = None

@property
def natural_key(self) -> str:
sha = hashlib.sha1(self.value.encode()).hexdigest()
key = super().natural_key
return key.replace(self.value, sha)


class DNSGPOSRecord(DNSLocation):
# RFC 1712
object_type: Literal["DNSGPOSRecord"] = "DNSGPOSRecord"
dns_record_type: Literal["GPOS"] = "GPOS"

_reverse_relation_names = {"hostname": "dns_gpos_records"}


class DNSLOCRecord(DNSLocation):
# RFC 1876
object_type: Literal["DNSLOCRecord"] = "DNSLOCRecord"
dns_record_type: Literal["LOC"] = "LOC"

horizontal_precision: int | None = None
vertical_precision: int | None = None
size: int | None = None

_reverse_relation_names = {"hostname": "dns_loc_records"}
4 changes: 4 additions & 0 deletions octopoes/octopoes/models/types.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@
DNSARecord,
DNSCAARecord,
DNSCNAMERecord,
DNSGPOSRecord,
DNSLOCRecord,
DNSMXRecord,
DNSNSRecord,
DNSPTRRecord,
Expand Down Expand Up @@ -93,6 +95,8 @@
| DNSSOARecord
| DNSCNAMERecord
| DNSCAARecord
| DNSLOCRecord
| DNSGPOSRecord
| ResolvedHostname
| NXDOMAIN
)
Expand Down