user: support HAR(HTTP Archive format) file

Makefile

@@ -147,19 +147,19 @@ $(KERN_OBJECTS_NOCORE): %.nocore: %.c \
 	.checkver_$(CMD_GO)
 	$(CMD_CLANG) \
 			$(EXTRA_CFLAGS_NOCORE) \
-    		$(BPFHEADER) \
+			$(BPFHEADER) \
 			-I $(KERN_SRC_PATH)/arch/$(LINUX_ARCH)/include \
 			-I $(KERN_BUILD_PATH)/arch/$(LINUX_ARCH)/include/generated \
 			-I $(KERN_SRC_PATH)/include \
 			-I $(KERN_SRC_PATH)/arch/$(LINUX_ARCH)/include/uapi \
 			-I $(KERN_BUILD_PATH)/arch/$(LINUX_ARCH)/include/generated/uapi \
 			-I $(KERN_SRC_PATH)/include/uapi \
 			-I $(KERN_BUILD_PATH)/include/generated/uapi \
-    		-c $< \
-    		-o - |$(CMD_LLC) \
-    		-march=bpf \
-    		-filetype=obj \
-    		-o $(subst kern/,user/bytecode/,$(subst .c,_noncore.o,$<))
+			-c $< \
+			-o - |$(CMD_LLC) \
+			-march=bpf \
+			-filetype=obj \
+			-o $(subst kern/,user/bytecode/,$(subst .c,_noncore.o,$<))
 	$(CMD_CLANG) \
 			$(EXTRA_CFLAGS_NOCORE) \
 			$(BPFHEADER) \
@@ -231,3 +231,13 @@ format:
 	@clang-format -i -style=$(STYLE) kern/openssl_masterkey_3.2.h
 	@clang-format -i -style=$(STYLE) kern/boringssl_masterkey.h
 	@clang-format -i -style=$(STYLE) utils/*.c
+
+
+# about string " -Wl,--no-gc-sections" in CGO_LDFLAGS, please see https://groups.google.com/g/golang-codereviews/c/ZnfJ5olFsnk.
+# if without "-Wl,--no-gc-sections", the error message is "runtime/cgo(.text): relocation target stderr not defined"
+.PHONY: gotest
+gotest:
+	CGO_CFLAGS='-O2 -g -I$(CURDIR)/lib/libpcap/ -Wl,--no-gc-sections' \
+	CGO_LDFLAGS='-O2 -g -L$(CURDIR)/lib/libpcap/ -lpcap -static' \
+	GOOS=linux GOARCH=$(GOARCH) CC=$(CMD_CC_PREFIX)$(CMD_CC) \
+	$(CMD_GO) test -v -race ./pkg/event_processor/...

README.md

@@ -4,13 +4,13 @@
 
 [![GitHub stars](https://img.shields.io/github/stars/gojue/ecapture.svg?label=Stars&logo=github)](https://github.com/gojue/ecapture)
 [![GitHub forks](https://img.shields.io/github/forks/gojue/ecapture?label=Forks&logo=github)](https://github.com/gojue/ecapture)
-[![CI](https://github.com/gojue/ecapture/actions/workflows/codeql-analysis.yml/badge.svg)](https://github.com/gojue/ecapture/actions/workflows/code-analysis.yml)
+[![CI](https://github.com/gojue/ecapture/actions/workflows/codeql-analysis.yml/badge.svg)](https://github.com/gojue/ecapture/blob/master/.github/workflows/codeql-analysis.yml)
 [![Github Version](https://img.shields.io/github/v/release/gojue/ecapture?display_name=tag&include_prereleases&sort=semver)](https://github.com/gojue/ecapture/releases)
+[![Docker](https://img.shields.io/docker/pulls/gojue/ecapture?style=flat&logo=docker)](https://github.com/gojue/ecapture)
 
 ### eCapture(旁观者): capture SSL/TLS text content without a CA certificate using eBPF.
 
-> **Note:**
->
+> [!IMPORTANT]  
 > Supports Linux/Android kernel versions x86_64 4.18 and above, **aarch64 5.5** and above.
 > Need ROOT permission.
 > Does not support Windows and macOS system.
@@ -27,7 +27,7 @@
   - [Modules](#modules)
     - [OpenSSL Module](#openssl-module)
     - [GoTLS Module](#gotls-module)
-    - [Other Modules](#bash-module)
+    - [Other Modules](#other-modules)
   - [Videos](#videos)
 - [Contributing](#contributing)
 - [Compilation](#compilation)
@@ -49,15 +49,15 @@
 
 ### ELF binary file
 
-> **Note**
+> [!TIP]
 > support Linux/Android x86_64/aarch64.
 
 Download ELF zip file [release](https://github.com/gojue/ecapture/releases) , unzip and use by
 command `sudo ecapture --help`.
 
 ### Docker image
 
-> **Note**
+> [!TIP]
 > Linux only.
 
 ```shell
@@ -178,6 +178,12 @@ The OpenSSL module supports three capture modes:
 Supported TLS encrypted http `1.0/1.1/2.0` over TCP, and http3 `QUIC` protocol over UDP.
 You can specify `-m pcap` or `-m pcapng` and use it in conjunction with `--pcapfile` and `-i` parameters. The default value for `--pcapfile` is `ecapture_openssl.pcapng`.
 
+```shell
+sudo ecapture tls -m pcap -i eth0 --pcapfile=ecapture.pcapng tcp port 443
+```
+
+This command saves captured plaintext data packets as a pcapng file, which can be viewed using `Wireshark`.
+
 ```shell
 sudo ecapture tls -m pcap -w ecap.pcapng -i ens160
 2024-09-15T06:54:12Z INF AppName="eCapture(旁观者)"
@@ -223,12 +229,6 @@ sudo ecapture tls -m pcap -w ecap.pcapng -i ens160
 
 Used `Wireshark` to open `ecap.pcapng` file to view the plaintext data packets.
 
-```shell
-sudo ecapture tls -m pcap -i eth0 --pcapfile=ecapture.pcapng tcp port 443
-```
-
-This command saves captured plaintext data packets as a pcapng file, which can be viewed using `Wireshark`.
-
 #### Keylog Mode
 
 You can specify `-m keylog` or `-m key` and use it in conjunction with the `--keylogfile` parameter, which defaults to `ecapture_masterkey.log`.
@@ -254,15 +254,6 @@ SSLKEYLOG information.)
 
 Similar to the OpenSSL module.
 
-#### check your server BTF config：
-
-```shell
-cfc4n@vm-server:~$# uname -r
-4.18.0-305.3.1.el8.x86_64
-cfc4n@vm-server:~$# cat /boot/config-`uname -r` | grep CONFIG_DEBUG_INFO_BTF
-CONFIG_DEBUG_INFO_BTF=y
-```
-
 #### gotls command
 
 capture tls text context.
@@ -298,6 +289,9 @@ such as `bash\mysqld\postgres` modules, you can use `ecapture -h` to view the li
 # Contributing
 See [CONTRIBUTING](./CONTRIBUTING.md) for details on submitting patches and the contribution workflow.
 
+The [eCapture roadmap](https://github.com/orgs/gojue/projects/1) lists many tasks to be developed, and you are also
+welcome to participate in the construction.
+
 # Compilation
 
 See [COMPILATION](./COMPILATION.md) for details on compiling the eCapture source code.

README_CN.md

@@ -4,16 +4,15 @@
 
 [![GitHub stars](https://img.shields.io/github/stars/gojue/ecapture.svg?label=Stars&logo=github)](https://github.com/gojue/ecapture)
 [![GitHub forks](https://img.shields.io/github/forks/gojue/ecapture?label=Forks&logo=github)](https://github.com/gojue/ecapture)
-[![CI](https://github.com/gojue/ecapture/actions/workflows/codeql-analysis.yml/badge.svg)](https://github.com/gojue/ecapture/actions/workflows/code-analysis.yml)
+[![CI](https://github.com/gojue/ecapture/actions/workflows/codeql-analysis.yml/badge.svg)](https://github.com/gojue/ecapture/blob/master/.github/workflows/codeql-analysis.yml)
 [![Github Version](https://img.shields.io/github/v/release/gojue/ecapture?display_name=tag&include_prereleases&sort=semver)](https://github.com/gojue/ecapture/releases)
+[![Docker](https://img.shields.io/docker/pulls/gojue/ecapture?style=flat&logo=docker)](https://github.com/gojue/ecapture)
+[![QQ 群](https://img.shields.io/badge/QQ_群-%2312B7F5?logo=tencent-qq&logoColor=white&style=flat)](https://qm.qq.com/cgi-bin/qm/qr?k=iCu561fq4zdbHVdntQLFV0Xugrnf7Hpv&jump_from=webapi&authKey=YamGv189Cg+KFdQt1Qnsw6GZlpx8BYA+G2WZFezohY4M03V+l0eElZWOhZj/wR/5)
 
 ### eCapture(旁观者): 基于eBPF技术实现SSL/TLS加密的明文捕获，无需CA证书。
 
-> **提醒：**
->
-> 支持Linux系统内核x86_64 4.18及以上版本，aarch64 5.5及以上版本；
-> 需要ROOT权限；
-> 不支持Windows、macOS系统；
+> [!IMPORTANT]
+> 支持Linux/Android系统内核x86_64 4.18及以上版本，aarch64 5.5及以上版本；不支持Windows、macOS系统；需要ROOT权限；
 
 ----
 <!-- MarkdownTOC autolink="true" -->
@@ -47,23 +46,21 @@ eCapture的中文名字为**旁观者**，即「**当局者迷，旁观者清**
 
 ### ELF可执行文件
 
-> **提醒**
->
+> [!TIP]
 > 支持 Linux/Android的x86_64/aarch64 CPU架构。
 
 下载 [release](https://github.com/gojue/ecapture/releases) 的二进制包，可直接使用。
 
 ### Docker容器镜像
 
-> **提醒**
->
+> [!TIP]
 > 仅支持Linux x86_64/aarch64。
 
 ```shell
 # 拉取镜像
 docker pull gojue/ecapture:latest
 # 运行
-docker run --rm --privileged=true --net=host -v ${宿主机文件路径}:${容器内路径} gojue/ecapture ARGS
+docker run --rm --privileged=true --net=host -v ${宿主机文件路径}:${容器内路径} gojue/ecapture ${启动参数}
 ```
 
 # 小试身手
@@ -268,7 +265,9 @@ eCapture 还支持其他模块，如`bash`、`mysql`、`nss`、`postgres`等，
 
 # 贡献
 
-参考 [CONTRIBUTING](./CONTRIBUTING.md)的介绍，提交缺陷、补丁、建议等，非常感谢。
+参考 [CONTRIBUTING](./CONTRIBUTING.md)
+的介绍，提交缺陷、补丁、建议等，非常感谢。另外，[eCapture路线图](https://github.com/orgs/gojue/projects/1)
+里列出了很多待开发的任务，也欢迎您一起建设。
 
 # 编译
 

cli/cmd/root.go

@@ -63,6 +63,13 @@ const (
 	eCaptureListenAddr = "localhost:28256"
 )
 
+// ZeroLog print level
+const (
+	eCaptureEventLevel        = zerolog.Level(88)
+	eCaptureEventName         = "DAT"
+	eCaptureEventConsoleColor = 35 // colorMagenta
+)
+
 // rootCmd represents the base command when called without any subcommands
 var rootCmd = &cobra.Command{
 	Use:        CliName,
@@ -132,7 +139,8 @@ type eventCollectorWriter struct {
 }
 
 func (e eventCollectorWriter) Write(p []byte) (n int, err error) {
-	return e.logger.Write(p)
+	e.logger.WithLevel(eCaptureEventLevel).Msgf("%s", p)
+	return len(p), nil
 }
 
 // setModConfig set module config
@@ -144,12 +152,18 @@ func setModConfig(globalConf config.BaseConfig, modConf config.IConfig) {
 	modConf.SetBTF(globalConf.BtfMode)
 	modConf.SetPerCpuMapSize(globalConf.PerCpuMapSize)
 	modConf.SetAddrType(loggerTypeStdout)
+	modConf.SetAppName(CliName)
+	modConf.SetAppVersion(GitVersion)
 }
 
 // initLogger init logger
 func initLogger(addr string, modConfig config.IConfig) zerolog.Logger {
 	var logger zerolog.Logger
 	var err error
+	// append zerolog Global variables
+	zerolog.FormattedLevels[eCaptureEventLevel] = eCaptureEventName
+	zerolog.LevelColors[eCaptureEventLevel] = eCaptureEventConsoleColor
+
 	consoleWriter := zerolog.ConsoleWriter{Out: os.Stdout, TimeFormat: time.RFC3339}
 	logger = zerolog.New(consoleWriter).With().Timestamp().Logger()
 	zerolog.SetGlobalLevel(zerolog.InfoLevel)
@@ -206,15 +220,14 @@ func runModule(modName string, modConfig config.IConfig) {
 	logger.Info().Str("Listen", globalConf.Listen).Send()
 	logger.Info().Str("logger", globalConf.LoggerAddr).Msg("eCapture running logs")
 	logger.Info().Str("eventCollector", globalConf.EventCollectorAddr).Msg("the file handler that receives the captured event")
-
 	var isReload bool
-	var reRloadConfig = make(chan config.IConfig, 10)
+	var reReloadConfig = make(chan config.IConfig, 10)
 
 	// listen http server
 	go func() {
 		logger.Info().Str("listen", globalConf.Listen).Send()
 		logger.Info().Msg("https server starting...You can update the configuration file via the HTTP interface.")
-		var ec = http.NewHttpServer(globalConf.Listen, reRloadConfig, logger)
+		var ec = http.NewHttpServer(globalConf.Listen, reReloadConfig, logger)
 		err = ec.Run()
 		if err != nil {
 			logger.Fatal().Err(err).Msg("http server start failed")
@@ -236,7 +249,7 @@ func runModule(modName string, modConfig config.IConfig) {
 
 	reload:
 		// 初始化
-		logger.Warn().Msg("========== module starting. ==========")
+		logger.Debug().Msg("========== module starting. ==========")
 		mod := modFunc()
 		ctx, cancelFun := context.WithCancel(context.TODO())
 		err = mod.Init(ctx, &logger, modConfig, ecw)
@@ -262,7 +275,7 @@ func runModule(modName string, modConfig config.IConfig) {
 				break
 			}
 			isReload = false
-		case rc, ok := <-reRloadConfig:
+		case rc, ok := <-reReloadConfig:
 			if !ok {
 				logger.Warn().Msg("reload config channel closed.")
 				isReload = false

pkg/event_processor/http_response.go

@@ -163,6 +163,7 @@ func (hr *HTTPResponse) Display() []byte {
 	//	for k, v := range hr.response.Header {
 	//		headerMap.WriteString(fmt.Sprintf("\t%s\t=>\t%s\n", k, v))
 	//	}
+	//b, err := http.ReadResponse(hr.bufReader, nil)
 	b, err := httputil.DumpResponse(hr.response, false)
 	if err != nil {
 		log.Println("[http response] DumpResponse error:", err)

pkg/event_processor/iworker.go

@@ -116,13 +116,26 @@ func (ew *eventWorker) Display() error {
 		b = []byte(hex.Dump(b))
 	}
 
+	//  判断包的类型，是不是HTTP 相关，如果是，则转化为har格式
+	var err error
+	switch ew.parser.ParserType() {
+	//case ParserTypeHttp2Request:
+	//case ParserTypeWebSocket:
+	//case ParserTypeHttpResponse:
+	//case ParserTypeHttpRequest:
+	//case ParserTypeHttp2Response:
+	//case ParserTypeNull:
+	//	fallthrough
+	default:
+		err = ew.writeToChan(fmt.Sprintf("UUID:%s, Name:%s, Type:%d, Length:%d\n%s\n", ew.UUID, ew.parser.Name(), ew.parser.ParserType(), len(b), b))
+	}
 	//iWorker只负责写入，不应该打印。
-	e := ew.writeToChan(fmt.Sprintf("UUID:%s, Name:%s, Type:%d, Length:%d\n%s\n", ew.UUID, ew.parser.Name(), ew.parser.ParserType(), len(b), b))
+
 	//ew.parser.Reset()
 	// 设定状态、重置包类型
 	ew.status = ProcessStateInit
 	ew.packetType = PacketTypeNull
-	return e
+	return err
 }
 
 func (ew *eventWorker) writeEvent(e event.IEventStruct) {
@@ -153,8 +166,6 @@ func (ew *eventWorker) Run() {
 			// 输出包
 			if ew.tickerCount > MaxTickerCount {
 				//ew.processor.GetLogger().Printf("eventWorker TickerCount > %d, event closed.", MaxTickerCount)
-				ew.processor.delWorkerByUUID(ew)
-
 				/*
 					When returned from delWorkerByUUID(), there are two possibilities:
 					1) no routine can touch it.
@@ -205,6 +216,9 @@ func (ew *eventWorker) Close() {
 	ew.ticker.Stop()
 	_ = ew.Display()
 	ew.tickerCount = 0
+
+	// 关闭时清除
+	ew.processor.delWorkerByUUID(ew)
 }
 
 func (ew *eventWorker) Get() {

pkg/event_processor/processor.go

@@ -18,6 +18,7 @@ import (
 	"errors"
 	"fmt"
 	"github.com/gojue/ecapture/user/event"
+	"github.com/rs/zerolog"
 	"io"
 	"sync"
 )
@@ -37,12 +38,15 @@ type EventProcessor struct {
 	// key为 PID+UID+COMMON等确定唯一的信息
 	workerQueue map[string]IWorker
 	// log
-	logger io.Writer
+	logger      *zerolog.Logger
+	eventLogger io.Writer
 
 	closeChan chan bool
 
 	// output model
-	isHex bool
+	isHex      bool
+	appName    string
+	appVersion string
 }
 
 func (ep *EventProcessor) GetLogger() io.Writer {
@@ -68,15 +72,15 @@ func (ep *EventProcessor) Serve() error {
 				return errors.Join(err, err1)
 			}
 		case s := <-ep.outComing:
-			_, _ = ep.GetLogger().Write([]byte(s))
+			_, _ = ep.eventLogger.Write([]byte(s))
 		case _ = <-ep.closeChan:
 			return nil
 		}
 	}
 }
 
 func (ep *EventProcessor) dispatch(e event.IEventStruct) error {
-	//ep.logger.Printf("event ID:%s", e.GetUUID())
+	ep.logger.Debug().Msgf("event ID:%s", e.GetUUID())
 	var uuid = e.GetUUID()
 	found, eWorker := ep.getWorkerByUUID(uuid)
 	if !found {
@@ -150,12 +154,17 @@ func (ep *EventProcessor) Close() error {
 	return nil
 }
 
-func NewEventProcessor(logger io.Writer, isHex bool) *EventProcessor {
+// NewEventProcessor 创建事件处理器
+func NewEventProcessor(logger *zerolog.Logger, eventLogger io.Writer, isHex bool, appName, appVer string) *EventProcessor {
 	var ep *EventProcessor
 	ep = &EventProcessor{}
+	// TODO 拆分为数据、日志两个通道
 	ep.logger = logger
+	ep.eventLogger = eventLogger
 	ep.isHex = isHex
 	ep.isClosed = false
+	ep.appName = appName
+	ep.appVersion = appVer
 	ep.init()
 	return ep
 }