Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/nfdump-ja4'
Browse files Browse the repository at this point in the history
  • Loading branch information
phaag committed Mar 3, 2024
2 parents 109c1c9 + 17ead4c commit 626be0b
Show file tree
Hide file tree
Showing 38 changed files with 1,630 additions and 231 deletions.
27 changes: 24 additions & 3 deletions ChangeLog
Original file line number Diff line number Diff line change
@@ -1,6 +1,27 @@
- 466c27b 2024-02-17 (HEAD -> master, origin/master, origin/HEAD) Merge remote-tracking branch 'origin/nfdump-filter-ng'
- c9fe29c 2024-03-03 (HEAD -> nfdump-ja4, origin/nfdump-ja4) Prepare nfstat for var length fields such as ja4
- 2a55c3b 2024-03-03 Commit
- 1e88803 2024-03-01 Implement ja4s
- 4a60672 2024-03-01 Update Readme
- c129b78 2024-03-01 Add ja4 Readme for license issue and add --enable-ja4, default no for building all ja4 modules
- 5c72cc8 2024-03-01 Add ja3 filter in nfdump-ja4
- 8ed9f34 2024-03-01 Add ja4_c
- 2ded6b5 2024-02-29 Implement ja_a and ja_b of ja
- ae92ef6 2024-02-29 Add ja4 files to start implementation
- 16148ed 2024-02-29 Add sha256
- a4f6a53 2024-02-26 Refrag ssl extension code
- 6ea1b58 2024-02-25 Add ALPN extension decoding to ssl.c
- 4115a8b 2024-02-25 Fix missing malloc() type casting
- 07136b3 2024-02-25 Use new ssl module for ja3
- 17d127e 2024-02-24 Debug ssl code - Client Hello
- 79e9b8a 2024-02-24 Defrag ssl and ja3 - Add plain ssl code
- 00e0e45 2024-02-24 (origin/master, origin/HEAD) Fix nfreader code
- eb5bef5 2024-02-22 Cleanup code to handle legacy sampler record
- a988a40 2024-02-18 Rework maxmind code
- 9a0dddf 2024-02-17 Fix library dependencies on some *nix
- 2c987ee 2024-02-17 The previous commit 466c27b is a merge with a working branch to remove the clumsy master_record_t in nfdump. This removes lot of legacy code and updates the code base. Furthermore, the filter engin has been replaced by a more flexible and faster one. The speed gain depends on the filter and the number of filter elements. The overall speed gain with this merge is 10% in average. Further improvements are planned with more worker threats. Please note that the filter syntax had some small changes such as 'icmp-type' is now 'icmp type' etc.
- 466c27b 2024-02-17 Merge remote-tracking branch 'origin/nfdump-filter-ng'
- bc08a0a 2024-02-17 Sync lz4 code
- 4f409e9 2024-02-17 Release v1.7.4
- 4f409e9 2024-02-17 (tag: v1.7.4) Release v1.7.4
- 4a7de41 2024-02-17 Fix compiler warnings for lz4
- 4e98a35 2024-02-17 Update lz4 code
- 519a052 2024-02-16 Fix compile issues
Expand Down Expand Up @@ -35,7 +56,7 @@
- 627565a 2024-01-29 Remove local m4 files
- f0d2750 2024-01-29 Fix err var in nfprofile
- dc1b1b2 2024-01-29 Fix #503 - Append records in nfprofile
- 1dcb7ae 2024-01-28 (origin/nfdump-filter-ng) Fix fmt time 1970
- 1dcb7ae 2024-01-28 Fix fmt time 1970
- c5ddbf0 2024-01-28 Fix fmt cores
- 427b08e 2024-01-28 Removed master record from nfgen. Update make check
- bbaef64 2024-01-26 master record removed from 1.7.x code
Expand Down
2 changes: 1 addition & 1 deletion Makefile.am
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
ACLOCAL_AMFLAGS = -I m4

SUBDIRS = src/lib src/output src/netflow src/collector src/maxmind src/nfdump src/nfcapd
SUBDIRS = src/lib src/output src/netflow src/collector src/maxmind src/decode src/nfdump src/nfcapd
SUBDIRS += src/nfanon src/nfexpire src/nfreplay . src src/test src/nfreader src/inline src/include

if SFLOW
Expand Down
3 changes: 3 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,9 @@ Build the flow-tools to nfdump converter; default is __NO__
Build nfprofile used by NfSen; default is __NO__
* __--enable-nftrack__
Build nftrack used by PortTracker; default is __NO__
* **--enable-ja4**
Enable all ja4 module; default is **NO**
See JA4-Fingerprinting [JA4 Fingerprinting](https://github.com/phaag/nfdump/blob/nfdump-ja4/src/decode/ja4/Readme.md) module.

Development and beta options

Expand Down
23 changes: 17 additions & 6 deletions configure.ac
Original file line number Diff line number Diff line change
Expand Up @@ -170,10 +170,17 @@ AM_CONDITIONAL([FT2NFDUMP], [test "x$build_ftconv" = "xyes"])
AC_ARG_ENABLE(maxmind,
[ --enable-maxmind Build geolookup for MaxMind GeoDB; default is NO])

build_maxmind="$enable_maxmind" # No dependencies
AS_IF([test "x$enable_maxmind" = xyes],
build_maxmind="$enable_maxmind", build_maxmind="no")
AM_CONDITIONAL([MAXMIND], [test "x$build_maxmind" = "xyes"])

#Needs tidy
AC_ARG_ENABLE(ja4,
[ --enable-ja4 Build with ja4 fingerprinting code; May require a license; default is NO])

AS_IF([test "x$enable_ja4" = xyes],
build_ja4="$enable_ja4", build_ja4="no")
AM_CONDITIONAL([JA4], [test "x$build_ja4" = "xyes"])

AC_ARG_ENABLE(nfprofile,
[ --enable-nfprofile Build nfprofile used by NfSen; default is NO])

Expand Down Expand Up @@ -616,19 +623,21 @@ AM_COND_IF([HAVE_DOXYGEN], AC_CONFIG_FILES([doc/Doxyfile]))
AC_CONFIG_FILES([doc/Makefile])
AC_OUTPUT

AC_CONFIG_FILES([Makefile src/lib/Makefile src/Makefile src/test/Makefile
src/output/Makefile src/netflow/Makefile src/collector/Makefile
src/maxmind/Makefile src/nfdump/Makefile src/nfcapd/Makefile src/nfexpire/Makefile
AC_CONFIG_FILES([Makefile src/lib/Makefile src/decode/Makefile
src/Makefile src/test/Makefile src/output/Makefile
src/netflow/Makefile src/collector/Makefile src/maxmind/Makefile
src/nfdump/Makefile src/nfcapd/Makefile src/nfexpire/Makefile
src/nfanon/Makefile src/nfreplay/Makefile src/nfreader/Makefile
src/inline/Makefile src/include/Makefile man/Makefile ])


if test "x$enable_ftconv" = "xyes"; then
AC_CONFIG_FILES([src/ft2nfdump/Makefile])
fi

if test "x$build_sflow" = "xyes"; then
AC_CONFIG_FILES([src/sflow/Makefile])
else
build_sflow="no"
fi

if test "x$build_nfpcapd" = "xyes"; then
Expand All @@ -655,11 +664,13 @@ echo " LIBS = $LIBS"
echo " Enable liblz4 = $use_lz4"
echo " Enable libbz2 = $use_bzip2"
echo " Enable libzstd = $use_zstd"
echo " Enable ja4 = $build_ja4"
echo " Build geolookup = $build_maxmind"
echo " Build sflow = $build_sflow"
echo " Build nfpcapd = $build_nfpcapd"
echo " Build flowtools conv = $build_ftconv"
echo " Build nfprofile = $build_nfprofile"
echo " Build ft2nfdump = $build_ftconv"
echo "----------------------------------"
echo ""
echo " You can run ./make now."
Expand Down
19 changes: 19 additions & 0 deletions src/decode/Makefile.am
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@

AM_CPPFLAGS = -I.. -I../include -I../inline -I../lib -Issl $(DEPS_CFLAGS)
AM_CFLAGS = -ggdb

LDADD = $(DEPS_LIBS)


# libnfdecode sources
decode = dns/dns.c dns/dns.h
decode += ssl/ssl.c ssl/ssl.h ja3/ja3.c ja3/ja3.h ja4/ja4.c ja4/ja4.h
decode += digest/md5.c digest/md5.h digest/sha256.c digest/sha256.h

if JA4
decode += ja4/ja4s.c ja4/ja4s.h
endif

noinst_LIBRARIES = libnfdecode.a

libnfdecode_a_SOURCES = $(decode)
File renamed without changes.
File renamed without changes.
222 changes: 222 additions & 0 deletions src/decode/digest/sha256.c
Original file line number Diff line number Diff line change
@@ -0,0 +1,222 @@
/*
* FIPS 180-2 SHA-224/256/384/512 implementation
* Last update: 02/02/2007
* Issue date: 04/30/2005
*
* Copyright (C) 2013, Con Kolivas <[email protected]>
* Copyright (C) 2005, 2007 Olivier Gay <[email protected]>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. Neither the name of the project nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/

#include "sha256.h"

#include <string.h>

#define SHA256_DIGEST_SIZE (256 / 8)
#define SHA256_BLOCK_SIZE (512 / 8)

#define SHFR(x, n) (x >> n)
#define ROTR(x, n) ((x >> n) | (x << ((sizeof(x) << 3) - n)))
#define CH(x, y, z) ((x & y) ^ (~x & z))
#define MAJ(x, y, z) ((x & y) ^ (x & z) ^ (y & z))

#define SHA256_F1(x) (ROTR(x, 2) ^ ROTR(x, 13) ^ ROTR(x, 22))
#define SHA256_F2(x) (ROTR(x, 6) ^ ROTR(x, 11) ^ ROTR(x, 25))
#define SHA256_F3(x) (ROTR(x, 7) ^ ROTR(x, 18) ^ SHFR(x, 3))
#define SHA256_F4(x) (ROTR(x, 17) ^ ROTR(x, 19) ^ SHFR(x, 10))

typedef struct {
unsigned int tot_len;
unsigned int len;
unsigned char block[2 * SHA256_BLOCK_SIZE];
uint32_t h[8];
} sha256_ctx;

#define UNPACK32(x, str) \
{ \
*((str) + 3) = (uint8_t)((x)); \
*((str) + 2) = (uint8_t)((x) >> 8); \
*((str) + 1) = (uint8_t)((x) >> 16); \
*((str) + 0) = (uint8_t)((x) >> 24); \
}

#define PACK32(str, x) \
{ *(x) = ((uint32_t) * ((str) + 3)) | ((uint32_t) * ((str) + 2) << 8) | ((uint32_t) * ((str) + 1) << 16) | ((uint32_t) * ((str) + 0) << 24); }

#define SHA256_SCR(i) \
{ w[i] = SHA256_F4(w[i - 2]) + w[i - 7] + SHA256_F3(w[i - 15]) + w[i - 16]; }

static uint32_t sha256_h0[8] = {0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19};

static uint32_t sha256_k[64] = {
0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, 0xd807aa98, 0x12835b01, 0x243185be,
0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa,
0x5cb0a9dc, 0x76f988da, 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, 0x27b70a85,
0x2e1b2138, 0x4d2c6dfc, 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3,
0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f,
0x682e6ff3, 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2};

/* SHA-256 functions */

static void sha256_init(sha256_ctx *ctx);

static void sha256_update(sha256_ctx *ctx, const unsigned char *message, unsigned int len);

static void sha256_final(sha256_ctx *ctx, unsigned char *digest);

static void sha256_transf(sha256_ctx *ctx, const unsigned char *message, unsigned int block_nb) {
uint32_t w[64];
uint32_t wv[8];
uint32_t t1, t2;
const unsigned char *sub_block;
int i;

int j;

for (i = 0; i < (int)block_nb; i++) {
sub_block = message + (i << 6);

for (j = 0; j < 16; j++) {
PACK32(&sub_block[j << 2], &w[j]);
}

for (j = 16; j < 64; j++) {
SHA256_SCR(j);
}

for (j = 0; j < 8; j++) {
wv[j] = ctx->h[j];
}

for (j = 0; j < 64; j++) {
t1 = wv[7] + SHA256_F2(wv[4]) + CH(wv[4], wv[5], wv[6]) + sha256_k[j] + w[j];
t2 = SHA256_F1(wv[0]) + MAJ(wv[0], wv[1], wv[2]);
wv[7] = wv[6];
wv[6] = wv[5];
wv[5] = wv[4];
wv[4] = wv[3] + t1;
wv[3] = wv[2];
wv[2] = wv[1];
wv[1] = wv[0];
wv[0] = t1 + t2;
}

for (j = 0; j < 8; j++) {
ctx->h[j] += wv[j];
}
}
}

static void sha256_init(sha256_ctx *ctx) {
int i;
for (i = 0; i < 8; i++) {
ctx->h[i] = sha256_h0[i];
}

ctx->len = 0;
ctx->tot_len = 0;
}

static void sha256_update(sha256_ctx *ctx, const unsigned char *message, unsigned int len) {
unsigned int block_nb;
unsigned int new_len, rem_len, tmp_len;
const unsigned char *shifted_message;

tmp_len = SHA256_BLOCK_SIZE - ctx->len;
rem_len = len < tmp_len ? len : tmp_len;

memcpy(&ctx->block[ctx->len], message, rem_len);

if (ctx->len + len < SHA256_BLOCK_SIZE) {
ctx->len += len;
return;
}

new_len = len - rem_len;
block_nb = new_len / SHA256_BLOCK_SIZE;

shifted_message = message + rem_len;

sha256_transf(ctx, ctx->block, 1);
sha256_transf(ctx, shifted_message, block_nb);

rem_len = new_len % SHA256_BLOCK_SIZE;

memcpy(ctx->block, &shifted_message[block_nb << 6], rem_len);

ctx->len = rem_len;
ctx->tot_len += (block_nb + 1) << 6;
}

static void sha256_final(sha256_ctx *ctx, unsigned char *digest) {
unsigned int block_nb;
unsigned int pm_len;
unsigned int len_b;

int i;

block_nb = (1 + ((SHA256_BLOCK_SIZE - 9) < (ctx->len % SHA256_BLOCK_SIZE)));

len_b = (ctx->tot_len + ctx->len) << 3;
pm_len = block_nb << 6;

memset(ctx->block + ctx->len, 0, pm_len - ctx->len);
ctx->block[ctx->len] = 0x80;
UNPACK32(len_b, ctx->block + pm_len - 4);

sha256_transf(ctx, ctx->block, block_nb);

for (i = 0; i < 8; i++) {
UNPACK32(ctx->h[i], &digest[i << 2]);
}
}

void sha256(const unsigned char *message, unsigned int len, unsigned char *digest) {
sha256_ctx ctx = {0};

sha256_init(&ctx);
sha256_update(&ctx, message, len);
sha256_final(&ctx, digest);
}

#ifdef MAIN
#include <stdio.h>

int main(int argc, char **argv) {
char *str = "b08abd37";

uint8_t shasum[32] = {0};

sha256((const unsigned char *)str, strlen(str), (unsigned char *)shasum);
for (int i = 0; i < 32; i++) {
printf("%x ", shasum[i]);
}
printf("\n");
return 0;
}

#endif
Loading

0 comments on commit 626be0b

Please sign in to comment.