From 28790849e051dc63e3df0aca5d834e001e020a77 Mon Sep 17 00:00:00 2001 From: "Valeriy V. Vorotyntsev" Date: Thu, 12 Oct 2023 00:48:44 +0300 Subject: [PATCH] Find certificates in "Server Hello" TLS packets Related issue: #11 --- rust/CHANGELOG.md | 9 +- rust/Cargo.lock | 8 +- rust/Cargo.toml | 2 +- rust/ja4/Cargo.toml | 2 +- .../ja4__insta@browsers-x509.pcapng.snap | 19 ++++ .../ja4__insta@http2-with-cookies.pcapng.snap | 22 +++++ .../snapshots/ja4__insta@latest.pcapng.snap | 38 ++++++++ .../src/snapshots/ja4__insta@ssh2.pcapng.snap | 34 +++++++ .../ja4__insta@tls-handshake.pcapng.snap | 19 ++++ rust/ja4/src/tls.rs | 89 ++++++++++--------- 10 files changed, 191 insertions(+), 51 deletions(-) diff --git a/rust/CHANGELOG.md b/rust/CHANGELOG.md index 12ab070..377006c 100644 --- a/rust/CHANGELOG.md +++ b/rust/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.15.1] - 2023-10-12 + +### Fixed + +- Don't skip X.509 certificates contained in "Server Hello" TLS packets. + ## [0.15.0] - 2023-10-08 ### Added @@ -19,6 +25,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Add Rust sources of `ja4` and `ja4x` CLI tools. -[unreleased]: https://github.com/FoxIO-LLC/ja4/compare/v0.15.0...HEAD +[unreleased]: https://github.com/FoxIO-LLC/ja4/compare/v0.15.1...HEAD +[0.15.1]: https://github.com/FoxIO-LLC/ja4/compare/v0.15.0...v0.15.1 [0.15.0]: https://github.com/FoxIO-LLC/ja4/compare/v0.14.0...v0.15.0 [0.14.0]: https://github.com/FoxIO-LLC/ja4/releases/tag/v0.14.0 diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 58b86de..7cdef65 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -531,7 +531,7 @@ checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" [[package]] name = "ja4" -version = "0.15.0" +version = "0.15.1" dependencies = [ "clap", "color-eyre", @@ -559,7 +559,7 @@ dependencies = [ [[package]] name = "ja4x" -version = "0.15.0" +version = "0.15.1" dependencies = [ "clap", "color-eyre", @@ -861,9 +861,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.19" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad977052201c6de01a8ef2aa3378c4bd23217a056337d1d6da40468d267a4fb0" +checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090" [[package]] name = "serde" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 17de08c..8961c10 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -3,7 +3,7 @@ members = ["ja4", "ja4x"] resolver = "2" [workspace.package] -version = "0.15.0" +version = "0.15.1" license = "LicenseRef-FoxIO-Proprietary" repository = "https://github.com/FoxIO-LLC/ja4" diff --git a/rust/ja4/Cargo.toml b/rust/ja4/Cargo.toml index c400581..2a98863 100644 --- a/rust/ja4/Cargo.toml +++ b/rust/ja4/Cargo.toml @@ -23,7 +23,7 @@ indexmap.workspace = true itertools.workspace = true ja4x = { path = "../ja4x" } owo-colors = "3.5" -rtshark = "2.6" +rtshark = "=2.6.0" # CAUTION: rtshark >= 2.7.0 breaks JA4 (TLS client) and JA4L-C/S fingerprints semver = "1.0" serde.workspace = true serde_json.workspace = true diff --git a/rust/ja4/src/snapshots/ja4__insta@browsers-x509.pcapng.snap b/rust/ja4/src/snapshots/ja4__insta@browsers-x509.pcapng.snap index 41504f3..4a87ffe 100644 --- a/rust/ja4/src/snapshots/ja4__insta@browsers-x509.pcapng.snap +++ b/rust/ja4/src/snapshots/ja4__insta@browsers-x509.pcapng.snap @@ -11,6 +11,25 @@ expression: output tls_server_name: edge.microsoft.com ja4: t13d1516h2_8daaf6152771_e5627efa2ab1 ja4s: t1206h2_c030_044dc9b3196d + tls_certs: + - x509: + - ja4x: a373a9f83c6b_2bab15409345_0f2217ba412e + issuerCountryName: US + issuerOrganizationName: Microsoft Corporation + issuerCommonName: Microsoft Azure TLS Issuing CA 05 + subjectCountryName: US + subjectStateOrProvinceName: WA + subjectLocalityName: Redmond + subjectOrganizationName: Microsoft Corporation + subjectCommonName: edge.microsoft.com + - ja4x: 7d5dbb3783b4_a373a9f83c6b_c34b04c10969 + issuerCountryName: US + issuerOrganizationName: DigiCert Inc + issuerOrganizationalUnit: www.digicert.com + issuerCommonName: DigiCert Global Root G2 + subjectCountryName: US + subjectOrganizationName: Microsoft Corporation + subjectCommonName: Microsoft Azure TLS Issuing CA 05 ja4l_c: 56_128 ja4l_s: 1907_112 - stream: 1 diff --git a/rust/ja4/src/snapshots/ja4__insta@http2-with-cookies.pcapng.snap b/rust/ja4/src/snapshots/ja4__insta@http2-with-cookies.pcapng.snap index ee79b51..161f84e 100644 --- a/rust/ja4/src/snapshots/ja4__insta@http2-with-cookies.pcapng.snap +++ b/rust/ja4/src/snapshots/ja4__insta@http2-with-cookies.pcapng.snap @@ -11,6 +11,28 @@ expression: output tls_server_name: youtube.com ja4: t13d1516h2_8daaf6152771_e5627efa2ab1 ja4s: t130200_1301_234ea6891581 + tls_certs: + - x509: + - ja4x: a373a9f83c6b_7022c563de38_2e3757343cb0 + issuerCountryName: US + issuerOrganizationName: Google Trust Services LLC + issuerCommonName: GTS CA 1C3 + subjectCommonName: '*.google.com' + - ja4x: a373a9f83c6b_a373a9f83c6b_5d71497f7704 + issuerCountryName: US + issuerOrganizationName: Google Trust Services LLC + issuerCommonName: GTS Root R1 + subjectCountryName: US + subjectOrganizationName: Google Trust Services LLC + subjectCommonName: GTS CA 1C3 + - ja4x: 7d5dbb3783b4_a373a9f83c6b_2fbee3f04f3b + issuerCountryName: BE + issuerOrganizationName: GlobalSign nv-sa + issuerOrganizationalUnit: Root CA + issuerCommonName: GlobalSign Root CA + subjectCountryName: US + subjectOrganizationName: Google Trust Services LLC + subjectCommonName: GTS Root R1 ja4l_c: 47_128 ja4l_s: 44840_117 http: diff --git a/rust/ja4/src/snapshots/ja4__insta@latest.pcapng.snap b/rust/ja4/src/snapshots/ja4__insta@latest.pcapng.snap index 73105bf..3783527 100644 --- a/rust/ja4/src/snapshots/ja4__insta@latest.pcapng.snap +++ b/rust/ja4/src/snapshots/ja4__insta@latest.pcapng.snap @@ -92,6 +92,25 @@ expression: output tls_server_name: ping-edge.smartscreen.microsoft.com ja4: t13d1516h2_8daaf6152771_e5627efa2ab1 ja4s: t120300_c030_09f674154ab3 + tls_certs: + - x509: + - ja4x: a373a9f83c6b_2bab15409345_0f2217ba412e + issuerCountryName: US + issuerOrganizationName: Microsoft Corporation + issuerCommonName: Microsoft Azure TLS Issuing CA 05 + subjectCountryName: US + subjectStateOrProvinceName: WA + subjectLocalityName: Redmond + subjectOrganizationName: Microsoft Corporation + subjectCommonName: smartscreen.microsoft.com + - ja4x: 7d5dbb3783b4_a373a9f83c6b_c34b04c10969 + issuerCountryName: US + issuerOrganizationName: DigiCert Inc + issuerOrganizationalUnit: www.digicert.com + issuerCommonName: DigiCert Global Root G2 + subjectCountryName: US + subjectOrganizationName: Microsoft Corporation + subjectCommonName: Microsoft Azure TLS Issuing CA 05 ja4l_c: 40_128 ja4l_s: 42103_109 - stream: 10 @@ -103,6 +122,25 @@ expression: output tls_server_name: data-edge.smartscreen.microsoft.com ja4: t13d1516h2_8daaf6152771_e5627efa2ab1 ja4s: t120300_c030_09f674154ab3 + tls_certs: + - x509: + - ja4x: a373a9f83c6b_2bab15409345_0f2217ba412e + issuerCountryName: US + issuerOrganizationName: Microsoft Corporation + issuerCommonName: Microsoft Azure TLS Issuing CA 05 + subjectCountryName: US + subjectStateOrProvinceName: WA + subjectLocalityName: Redmond + subjectOrganizationName: Microsoft Corporation + subjectCommonName: smartscreen.microsoft.com + - ja4x: 7d5dbb3783b4_a373a9f83c6b_c34b04c10969 + issuerCountryName: US + issuerOrganizationName: DigiCert Inc + issuerOrganizationalUnit: www.digicert.com + issuerCommonName: DigiCert Global Root G2 + subjectCountryName: US + subjectOrganizationName: Microsoft Corporation + subjectCommonName: Microsoft Azure TLS Issuing CA 05 ja4l_c: 61_128 ja4l_s: 53595_109 diff --git a/rust/ja4/src/snapshots/ja4__insta@ssh2.pcapng.snap b/rust/ja4/src/snapshots/ja4__insta@ssh2.pcapng.snap index 686bc2f..d86a95c 100644 --- a/rust/ja4/src/snapshots/ja4__insta@ssh2.pcapng.snap +++ b/rust/ja4/src/snapshots/ja4__insta@ssh2.pcapng.snap @@ -73,6 +73,25 @@ expression: output tls_server_name: self.events.data.microsoft.com ja4: t12d190800_d83cc789557e_7af1ed941c26 ja4s: t120300_c030_09f674154ab3 + tls_certs: + - x509: + - ja4x: a373a9f83c6b_2bab15409345_0f2217ba412e + issuerCountryName: US + issuerOrganizationName: Microsoft Corporation + issuerCommonName: Microsoft Azure TLS Issuing CA 05 + subjectCountryName: US + subjectStateOrProvinceName: WA + subjectLocalityName: Redmond + subjectOrganizationName: Microsoft Corporation + subjectCommonName: '*.events.data.microsoft.com' + - ja4x: 7d5dbb3783b4_a373a9f83c6b_c34b04c10969 + issuerCountryName: US + issuerOrganizationName: DigiCert Inc + issuerOrganizationalUnit: www.digicert.com + issuerCommonName: DigiCert Global Root G2 + subjectCountryName: US + subjectOrganizationName: Microsoft Corporation + subjectCommonName: Microsoft Azure TLS Issuing CA 05 ja4l_c: 46_128 ja4l_s: 49308_110 - stream: 12 @@ -84,6 +103,21 @@ expression: output tls_server_name: www.bing.com ja4: t13d1516h2_8daaf6152771_e5627efa2ab1 ja4s: t1205h2_c030_015e35fdd027 + tls_certs: + - x509: + - ja4x: a373a9f83c6b_7022c563de38_0ce9ea683d50 + issuerCountryName: US + issuerOrganizationName: Microsoft Corporation + issuerCommonName: Microsoft RSA TLS CA 02 + subjectCommonName: www.bing.com + - ja4x: 7d5dbb3783b4_a373a9f83c6b_44440d41940c + issuerCountryName: IE + issuerOrganizationName: Baltimore + issuerOrganizationalUnit: CyberTrust + issuerCommonName: Baltimore CyberTrust Root + subjectCountryName: US + subjectOrganizationName: Microsoft Corporation + subjectCommonName: Microsoft RSA TLS CA 02 ja4l_c: 55_128 ja4l_s: 3217_119 - stream: 13 diff --git a/rust/ja4/src/snapshots/ja4__insta@tls-handshake.pcapng.snap b/rust/ja4/src/snapshots/ja4__insta@tls-handshake.pcapng.snap index 12950b9..6481b5d 100644 --- a/rust/ja4/src/snapshots/ja4__insta@tls-handshake.pcapng.snap +++ b/rust/ja4/src/snapshots/ja4__insta@tls-handshake.pcapng.snap @@ -482,6 +482,25 @@ expression: output tls_server_name: login.live.com ja4: t13d1516h2_8daaf6152771_e5627efa2ab1 ja4s: t120400_c030_4e8089b08790 + tls_certs: + - x509: + - ja4x: a373a9f83c6b_2bab15409345_7bf9a7bf7029 + issuerCountryName: US + issuerOrganizationName: DigiCert Inc + issuerCommonName: DigiCert SHA2 Secure Server CA + subjectCountryName: US + subjectStateOrProvinceName: Washington + subjectLocalityName: Redmond + subjectOrganizationName: Microsoft Corporation + subjectCommonName: login.live.com + - ja4x: 7d5dbb3783b4_a373a9f83c6b_44440d41940c + issuerCountryName: US + issuerOrganizationName: DigiCert Inc + issuerOrganizationalUnit: www.digicert.com + issuerCommonName: DigiCert Global Root CA + subjectCountryName: US + subjectOrganizationName: DigiCert Inc + subjectCommonName: DigiCert SHA2 Secure Server CA - stream: 44 transport: tcp src: 192.168.1.168 diff --git a/rust/ja4/src/tls.rs b/rust/ja4/src/tls.rs index 8b2015c..9ba5226 100644 --- a/rust/ja4/src/tls.rs +++ b/rust/ja4/src/tls.rs @@ -46,59 +46,60 @@ impl Stream { else { return Ok(()); }; - let tls_handshake_type = tls.find("tls.handshake.type").unwrap(); const CLIENT_HELLO: &str = "1"; const SERVER_HELLO: &str = "2"; const CERTIFICATE: &str = "11"; - match tls_handshake_type.value() { - CLIENT_HELLO => { - debug_assert_eq!( - tls_handshake_type.display(), - "Handshake Type: Client Hello (1)", - "packet={}", - pkt.num - ); - // We only process a single TLS Client Hello packet per stream. - if self.client.is_none() { - self.client = Some(ClientStats::new(pkt, &tls, store_pkt_num)?); + for tls_handshake_type in tls.fields("tls.handshake.type") { + match tls_handshake_type.value() { + CLIENT_HELLO => { + debug_assert_eq!( + tls_handshake_type.display(), + "Handshake Type: Client Hello (1)", + "packet={}", + pkt.num + ); + // We only process a single TLS Client Hello packet per stream. + if self.client.is_none() { + self.client = Some(ClientStats::new(pkt, &tls, store_pkt_num)?); + } } - } - SERVER_HELLO => { - debug_assert_eq!( - tls_handshake_type.display(), - "Handshake Type: Server Hello (2)" - ); - // We only need data from a single TLS Server Hello packet per stream. - if self.server.is_none() { - self.server = ServerStats::try_new(pkt, &tls, store_pkt_num)?; + SERVER_HELLO => { + debug_assert_eq!( + tls_handshake_type.display(), + "Handshake Type: Server Hello (2)" + ); + // We only need data from a single TLS Server Hello packet per stream. + if self.server.is_none() { + self.server = ServerStats::try_new(pkt, &tls, store_pkt_num)?; + } } - } - CERTIFICATE => { - debug_assert_eq!( - tls_handshake_type.display(), - "Handshake Type: Certificate (11)" - ); - - let mut recs = Vec::new(); - for hexdump in tls.values("tls.handshake.certificate") { - let der = hexdump - .split(':') - .map(|s| u8::from_str_radix(s, 16).map_err(|e| e.into())) - .collect::>>()?; - let (rem, x509) = X509Certificate::from_der(&der)?; - debug_assert!(rem.is_empty()); - recs.push(ja4x::X509Rec::from(x509)); + CERTIFICATE => { + debug_assert_eq!( + tls_handshake_type.display(), + "Handshake Type: Certificate (11)" + ); + + let mut recs = Vec::new(); + for hexdump in tls.values("tls.handshake.certificate") { + let der = hexdump + .split(':') + .map(|s| u8::from_str_radix(s, 16).map_err(|e| e.into())) + .collect::>>()?; + let (rem, x509) = X509Certificate::from_der(&der)?; + debug_assert!(rem.is_empty()); + recs.push(ja4x::X509Rec::from(x509)); + } + debug_assert!(!recs.is_empty()); + + self.x509.push(X509Stats { + packet: store_pkt_num.then_some(pkt.num), + recs, + }); } - debug_assert!(!recs.is_empty()); - - self.x509.push(X509Stats { - packet: store_pkt_num.then_some(pkt.num), - recs, - }); + _ => {} } - _ => {} } Ok(()) }