Merge pull request #14 from vvv/vvv.issue-11

Find certificates in "Server Hello" TLS packets
FoxIO-LLC · Oct 11, 2023 · e192c81 · e192c81
2 parents 41c2c89 + 2879084
commit e192c81
Show file tree

Hide file tree

Showing 10 changed files with 191 additions and 51 deletions.
diff --git a/rust/CHANGELOG.md b/rust/CHANGELOG.md
@@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.15.1] - 2023-10-12
+
+### Fixed
+
+- Don't skip X.509 certificates contained in "Server Hello" TLS packets.
+
 ## [0.15.0] - 2023-10-08
 
 ### Added
@@ -19,6 +25,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Add Rust sources of `ja4` and `ja4x` CLI tools.
 
-[unreleased]: https://github.com/FoxIO-LLC/ja4/compare/v0.15.0...HEAD
+[unreleased]: https://github.com/FoxIO-LLC/ja4/compare/v0.15.1...HEAD
+[0.15.1]: https://github.com/FoxIO-LLC/ja4/compare/v0.15.0...v0.15.1
 [0.15.0]: https://github.com/FoxIO-LLC/ja4/compare/v0.14.0...v0.15.0
 [0.14.0]: https://github.com/FoxIO-LLC/ja4/releases/tag/v0.14.0
diff --git a/rust/Cargo.lock b/rust/Cargo.lock
diff --git a/rust/Cargo.toml b/rust/Cargo.toml
@@ -3,7 +3,7 @@ members = ["ja4", "ja4x"]
 resolver = "2"
 
 [workspace.package]
-version = "0.15.0"
+version = "0.15.1"
 license = "LicenseRef-FoxIO-Proprietary"
 repository = "https://github.com/FoxIO-LLC/ja4"
 

diff --git a/rust/ja4/Cargo.toml b/rust/ja4/Cargo.toml
@@ -23,7 +23,7 @@ indexmap.workspace = true
 itertools.workspace = true
 ja4x = { path = "../ja4x" }
 owo-colors = "3.5"
-rtshark = "2.6"
+rtshark = "=2.6.0"  # CAUTION: rtshark >= 2.7.0 breaks JA4 (TLS client) and JA4L-C/S fingerprints
 semver = "1.0"
 serde.workspace = true
 serde_json.workspace = true

diff --git a/rust/ja4/src/snapshots/[email protected] b/rust/ja4/src/snapshots/[email protected]
@@ -11,6 +11,25 @@ expression: output
   tls_server_name: edge.microsoft.com
   ja4: t13d1516h2_8daaf6152771_e5627efa2ab1
   ja4s: t1206h2_c030_044dc9b3196d
+  tls_certs:
+  - x509:
+    - ja4x: a373a9f83c6b_2bab15409345_0f2217ba412e
+      issuerCountryName: US
+      issuerOrganizationName: Microsoft Corporation
+      issuerCommonName: Microsoft Azure TLS Issuing CA 05
+      subjectCountryName: US
+      subjectStateOrProvinceName: WA
+      subjectLocalityName: Redmond
+      subjectOrganizationName: Microsoft Corporation
+      subjectCommonName: edge.microsoft.com
+    - ja4x: 7d5dbb3783b4_a373a9f83c6b_c34b04c10969
+      issuerCountryName: US
+      issuerOrganizationName: DigiCert Inc
+      issuerOrganizationalUnit: www.digicert.com
+      issuerCommonName: DigiCert Global Root G2
+      subjectCountryName: US
+      subjectOrganizationName: Microsoft Corporation
+      subjectCommonName: Microsoft Azure TLS Issuing CA 05
   ja4l_c: 56_128
   ja4l_s: 1907_112
 - stream: 1

diff --git a/rust/ja4/src/snapshots/[email protected] b/rust/ja4/src/snapshots/[email protected]
@@ -11,6 +11,28 @@ expression: output
   tls_server_name: youtube.com
   ja4: t13d1516h2_8daaf6152771_e5627efa2ab1
   ja4s: t130200_1301_234ea6891581
+  tls_certs:
+  - x509:
+    - ja4x: a373a9f83c6b_7022c563de38_2e3757343cb0
+      issuerCountryName: US
+      issuerOrganizationName: Google Trust Services LLC
+      issuerCommonName: GTS CA 1C3
+      subjectCommonName: '*.google.com'
+    - ja4x: a373a9f83c6b_a373a9f83c6b_5d71497f7704
+      issuerCountryName: US
+      issuerOrganizationName: Google Trust Services LLC
+      issuerCommonName: GTS Root R1
+      subjectCountryName: US
+      subjectOrganizationName: Google Trust Services LLC
+      subjectCommonName: GTS CA 1C3
+    - ja4x: 7d5dbb3783b4_a373a9f83c6b_2fbee3f04f3b
+      issuerCountryName: BE
+      issuerOrganizationName: GlobalSign nv-sa
+      issuerOrganizationalUnit: Root CA
+      issuerCommonName: GlobalSign Root CA
+      subjectCountryName: US
+      subjectOrganizationName: Google Trust Services LLC
+      subjectCommonName: GTS Root R1
   ja4l_c: 47_128
   ja4l_s: 44840_117
   http:

diff --git a/rust/ja4/src/snapshots/[email protected] b/rust/ja4/src/snapshots/[email protected]
@@ -92,6 +92,25 @@ expression: output
   tls_server_name: ping-edge.smartscreen.microsoft.com
   ja4: t13d1516h2_8daaf6152771_e5627efa2ab1
   ja4s: t120300_c030_09f674154ab3
+  tls_certs:
+  - x509:
+    - ja4x: a373a9f83c6b_2bab15409345_0f2217ba412e
+      issuerCountryName: US
+      issuerOrganizationName: Microsoft Corporation
+      issuerCommonName: Microsoft Azure TLS Issuing CA 05
+      subjectCountryName: US
+      subjectStateOrProvinceName: WA
+      subjectLocalityName: Redmond
+      subjectOrganizationName: Microsoft Corporation
+      subjectCommonName: smartscreen.microsoft.com
+    - ja4x: 7d5dbb3783b4_a373a9f83c6b_c34b04c10969
+      issuerCountryName: US
+      issuerOrganizationName: DigiCert Inc
+      issuerOrganizationalUnit: www.digicert.com
+      issuerCommonName: DigiCert Global Root G2
+      subjectCountryName: US
+      subjectOrganizationName: Microsoft Corporation
+      subjectCommonName: Microsoft Azure TLS Issuing CA 05
   ja4l_c: 40_128
   ja4l_s: 42103_109
 - stream: 10
@@ -103,6 +122,25 @@ expression: output
   tls_server_name: data-edge.smartscreen.microsoft.com
   ja4: t13d1516h2_8daaf6152771_e5627efa2ab1
   ja4s: t120300_c030_09f674154ab3
+  tls_certs:
+  - x509:
+    - ja4x: a373a9f83c6b_2bab15409345_0f2217ba412e
+      issuerCountryName: US
+      issuerOrganizationName: Microsoft Corporation
+      issuerCommonName: Microsoft Azure TLS Issuing CA 05
+      subjectCountryName: US
+      subjectStateOrProvinceName: WA
+      subjectLocalityName: Redmond
+      subjectOrganizationName: Microsoft Corporation
+      subjectCommonName: smartscreen.microsoft.com
+    - ja4x: 7d5dbb3783b4_a373a9f83c6b_c34b04c10969
+      issuerCountryName: US
+      issuerOrganizationName: DigiCert Inc
+      issuerOrganizationalUnit: www.digicert.com
+      issuerCommonName: DigiCert Global Root G2
+      subjectCountryName: US
+      subjectOrganizationName: Microsoft Corporation
+      subjectCommonName: Microsoft Azure TLS Issuing CA 05
   ja4l_c: 61_128
   ja4l_s: 53595_109
 
diff --git a/rust/ja4/src/snapshots/[email protected] b/rust/ja4/src/snapshots/[email protected]
@@ -73,6 +73,25 @@ expression: output
   tls_server_name: self.events.data.microsoft.com
   ja4: t12d190800_d83cc789557e_7af1ed941c26
   ja4s: t120300_c030_09f674154ab3
+  tls_certs:
+  - x509:
+    - ja4x: a373a9f83c6b_2bab15409345_0f2217ba412e
+      issuerCountryName: US
+      issuerOrganizationName: Microsoft Corporation
+      issuerCommonName: Microsoft Azure TLS Issuing CA 05
+      subjectCountryName: US
+      subjectStateOrProvinceName: WA
+      subjectLocalityName: Redmond
+      subjectOrganizationName: Microsoft Corporation
+      subjectCommonName: '*.events.data.microsoft.com'
+    - ja4x: 7d5dbb3783b4_a373a9f83c6b_c34b04c10969
+      issuerCountryName: US
+      issuerOrganizationName: DigiCert Inc
+      issuerOrganizationalUnit: www.digicert.com
+      issuerCommonName: DigiCert Global Root G2
+      subjectCountryName: US
+      subjectOrganizationName: Microsoft Corporation
+      subjectCommonName: Microsoft Azure TLS Issuing CA 05
   ja4l_c: 46_128
   ja4l_s: 49308_110
 - stream: 12
@@ -84,6 +103,21 @@ expression: output
   tls_server_name: www.bing.com
   ja4: t13d1516h2_8daaf6152771_e5627efa2ab1
   ja4s: t1205h2_c030_015e35fdd027
+  tls_certs:
+  - x509:
+    - ja4x: a373a9f83c6b_7022c563de38_0ce9ea683d50
+      issuerCountryName: US
+      issuerOrganizationName: Microsoft Corporation
+      issuerCommonName: Microsoft RSA TLS CA 02
+      subjectCommonName: www.bing.com
+    - ja4x: 7d5dbb3783b4_a373a9f83c6b_44440d41940c
+      issuerCountryName: IE
+      issuerOrganizationName: Baltimore
+      issuerOrganizationalUnit: CyberTrust
+      issuerCommonName: Baltimore CyberTrust Root
+      subjectCountryName: US
+      subjectOrganizationName: Microsoft Corporation
+      subjectCommonName: Microsoft RSA TLS CA 02
   ja4l_c: 55_128
   ja4l_s: 3217_119
 - stream: 13

diff --git a/rust/ja4/src/snapshots/[email protected] b/rust/ja4/src/snapshots/[email protected]
@@ -482,6 +482,25 @@ expression: output
   tls_server_name: login.live.com
   ja4: t13d1516h2_8daaf6152771_e5627efa2ab1
   ja4s: t120400_c030_4e8089b08790
+  tls_certs:
+  - x509:
+    - ja4x: a373a9f83c6b_2bab15409345_7bf9a7bf7029
+      issuerCountryName: US
+      issuerOrganizationName: DigiCert Inc
+      issuerCommonName: DigiCert SHA2 Secure Server CA
+      subjectCountryName: US
+      subjectStateOrProvinceName: Washington
+      subjectLocalityName: Redmond
+      subjectOrganizationName: Microsoft Corporation
+      subjectCommonName: login.live.com
+    - ja4x: 7d5dbb3783b4_a373a9f83c6b_44440d41940c
+      issuerCountryName: US
+      issuerOrganizationName: DigiCert Inc
+      issuerOrganizationalUnit: www.digicert.com
+      issuerCommonName: DigiCert Global Root CA
+      subjectCountryName: US
+      subjectOrganizationName: DigiCert Inc
+      subjectCommonName: DigiCert SHA2 Secure Server CA
 - stream: 44
   transport: tcp
   src: 192.168.1.168

diff --git a/rust/ja4/src/tls.rs b/rust/ja4/src/tls.rs
@@ -46,59 +46,60 @@ impl Stream {
         else {
             return Ok(());
         };
-        let tls_handshake_type = tls.find("tls.handshake.type").unwrap();
 
         const CLIENT_HELLO: &str = "1";
         const SERVER_HELLO: &str = "2";
         const CERTIFICATE: &str = "11";
 
-        match tls_handshake_type.value() {
-            CLIENT_HELLO => {
-                debug_assert_eq!(
-                    tls_handshake_type.display(),
-                    "Handshake Type: Client Hello (1)",
-                    "packet={}",
-                    pkt.num
-                );
-                // We only process a single TLS Client Hello packet per stream.
-                if self.client.is_none() {
-                    self.client = Some(ClientStats::new(pkt, &tls, store_pkt_num)?);
+        for tls_handshake_type in tls.fields("tls.handshake.type") {
+            match tls_handshake_type.value() {
+                CLIENT_HELLO => {
+                    debug_assert_eq!(
+                        tls_handshake_type.display(),
+                        "Handshake Type: Client Hello (1)",
+                        "packet={}",
+                        pkt.num
+                    );
+                    // We only process a single TLS Client Hello packet per stream.
+                    if self.client.is_none() {
+                        self.client = Some(ClientStats::new(pkt, &tls, store_pkt_num)?);
+                    }
                 }
-            }
-            SERVER_HELLO => {
-                debug_assert_eq!(
-                    tls_handshake_type.display(),
-                    "Handshake Type: Server Hello (2)"
-                );
-                // We only need data from a single TLS Server Hello packet per stream.
-                if self.server.is_none() {
-                    self.server = ServerStats::try_new(pkt, &tls, store_pkt_num)?;
+                SERVER_HELLO => {
+                    debug_assert_eq!(
+                        tls_handshake_type.display(),
+                        "Handshake Type: Server Hello (2)"
+                    );
+                    // We only need data from a single TLS Server Hello packet per stream.
+                    if self.server.is_none() {
+                        self.server = ServerStats::try_new(pkt, &tls, store_pkt_num)?;
+                    }
                 }
-            }
-            CERTIFICATE => {
-                debug_assert_eq!(
-                    tls_handshake_type.display(),
-                    "Handshake Type: Certificate (11)"
-                );
-
-                let mut recs = Vec::new();
-                for hexdump in tls.values("tls.handshake.certificate") {
-                    let der = hexdump
-                        .split(':')
-                        .map(|s| u8::from_str_radix(s, 16).map_err(|e| e.into()))
-                        .collect::<Result<Vec<_>>>()?;
-                    let (rem, x509) = X509Certificate::from_der(&der)?;
-                    debug_assert!(rem.is_empty());
-                    recs.push(ja4x::X509Rec::from(x509));
+                CERTIFICATE => {
+                    debug_assert_eq!(
+                        tls_handshake_type.display(),
+                        "Handshake Type: Certificate (11)"
+                    );
+
+                    let mut recs = Vec::new();
+                    for hexdump in tls.values("tls.handshake.certificate") {
+                        let der = hexdump
+                            .split(':')
+                            .map(|s| u8::from_str_radix(s, 16).map_err(|e| e.into()))
+                            .collect::<Result<Vec<_>>>()?;
+                        let (rem, x509) = X509Certificate::from_der(&der)?;
+                        debug_assert!(rem.is_empty());
+                        recs.push(ja4x::X509Rec::from(x509));
+                    }
+                    debug_assert!(!recs.is_empty());
+
+                    self.x509.push(X509Stats {
+                        packet: store_pkt_num.then_some(pkt.num),
+                        recs,
+                    });
                 }
-                debug_assert!(!recs.is_empty());
-
-                self.x509.push(X509Stats {
-                    packet: store_pkt_num.then_some(pkt.num),
-                    recs,
-                });
+                _ => {}
             }
-            _ => {}
         }
         Ok(())
     }