From bf309829e3a9bfb9e6b19fffe0f2b0916714851c Mon Sep 17 00:00:00 2001 From: CFC4N Date: Tue, 17 Sep 2024 00:12:53 +0800 Subject: [PATCH 1/8] user: support HAR(HTTP Archive format) file Signed-off-by: CFC4N --- cli/cmd/root.go | 25 +- pkg/event_processor/http_response.go | 1 + pkg/event_processor/iworker.go | 22 +- pkg/event_processor/processor.go | 19 +- pkg/event_processor/processor_test.go | 2 +- pkg/har/ctx.go | 76 ++ pkg/har/har.go | 824 ++++++++++++++++++++++ pkg/har/har_handlers.go | 99 +++ pkg/har/har_handlers_test.go | 141 ++++ pkg/har/har_test.go | 954 ++++++++++++++++++++++++++ user/config/iconfig.go | 23 + user/module/imodule.go | 13 +- 12 files changed, 2176 insertions(+), 23 deletions(-) create mode 100644 pkg/har/ctx.go create mode 100644 pkg/har/har.go create mode 100644 pkg/har/har_handlers.go create mode 100644 pkg/har/har_handlers_test.go create mode 100644 pkg/har/har_test.go diff --git a/cli/cmd/root.go b/cli/cmd/root.go index 3f861a08d..4ac27f347 100644 --- a/cli/cmd/root.go +++ b/cli/cmd/root.go @@ -63,6 +63,13 @@ const ( eCaptureListenAddr = "localhost:28256" ) +// ZeroLog print level +const ( + eCaptureEventLevel = zerolog.Level(88) + eCaptureEventName = "DAT" + eCaptureEventConsoleColor = 35 // colorMagenta +) + // rootCmd represents the base command when called without any subcommands var rootCmd = &cobra.Command{ Use: CliName, @@ -132,7 +139,8 @@ type eventCollectorWriter struct { } func (e eventCollectorWriter) Write(p []byte) (n int, err error) { - return e.logger.Write(p) + e.logger.WithLevel(eCaptureEventLevel).Msgf("%s", p) + return len(p), nil } // setModConfig set module config @@ -144,12 +152,18 @@ func setModConfig(globalConf config.BaseConfig, modConf config.IConfig) { modConf.SetBTF(globalConf.BtfMode) modConf.SetPerCpuMapSize(globalConf.PerCpuMapSize) modConf.SetAddrType(loggerTypeStdout) + modConf.SetAppName(CliName) + modConf.SetAppVersion(GitVersion) } // initLogger init logger func initLogger(addr string, modConfig config.IConfig) zerolog.Logger { var logger zerolog.Logger var err error + // append zerolog Global variables + zerolog.FormattedLevels[eCaptureEventLevel] = eCaptureEventName + zerolog.LevelColors[eCaptureEventLevel] = eCaptureEventConsoleColor + consoleWriter := zerolog.ConsoleWriter{Out: os.Stdout, TimeFormat: time.RFC3339} logger = zerolog.New(consoleWriter).With().Timestamp().Logger() zerolog.SetGlobalLevel(zerolog.InfoLevel) @@ -206,15 +220,14 @@ func runModule(modName string, modConfig config.IConfig) { logger.Info().Str("Listen", globalConf.Listen).Send() logger.Info().Str("logger", globalConf.LoggerAddr).Msg("eCapture running logs") logger.Info().Str("eventCollector", globalConf.EventCollectorAddr).Msg("the file handler that receives the captured event") - var isReload bool - var reRloadConfig = make(chan config.IConfig, 10) + var reReloadConfig = make(chan config.IConfig, 10) // listen http server go func() { logger.Info().Str("listen", globalConf.Listen).Send() logger.Info().Msg("https server starting...You can update the configuration file via the HTTP interface.") - var ec = http.NewHttpServer(globalConf.Listen, reRloadConfig, logger) + var ec = http.NewHttpServer(globalConf.Listen, reReloadConfig, logger) err = ec.Run() if err != nil { logger.Fatal().Err(err).Msg("http server start failed") @@ -236,7 +249,7 @@ func runModule(modName string, modConfig config.IConfig) { reload: // 初始化 - logger.Warn().Msg("========== module starting. ==========") + logger.Debug().Msg("========== module starting. ==========") mod := modFunc() ctx, cancelFun := context.WithCancel(context.TODO()) err = mod.Init(ctx, &logger, modConfig, ecw) @@ -262,7 +275,7 @@ func runModule(modName string, modConfig config.IConfig) { break } isReload = false - case rc, ok := <-reRloadConfig: + case rc, ok := <-reReloadConfig: if !ok { logger.Warn().Msg("reload config channel closed.") isReload = false diff --git a/pkg/event_processor/http_response.go b/pkg/event_processor/http_response.go index 6751b8068..e65227f93 100644 --- a/pkg/event_processor/http_response.go +++ b/pkg/event_processor/http_response.go @@ -163,6 +163,7 @@ func (hr *HTTPResponse) Display() []byte { // for k, v := range hr.response.Header { // headerMap.WriteString(fmt.Sprintf("\t%s\t=>\t%s\n", k, v)) // } + //b, err := http.ReadResponse(hr.bufReader, nil) b, err := httputil.DumpResponse(hr.response, false) if err != nil { log.Println("[http response] DumpResponse error:", err) diff --git a/pkg/event_processor/iworker.go b/pkg/event_processor/iworker.go index 0eed31c31..0363d06dc 100644 --- a/pkg/event_processor/iworker.go +++ b/pkg/event_processor/iworker.go @@ -116,13 +116,26 @@ func (ew *eventWorker) Display() error { b = []byte(hex.Dump(b)) } + // 判断包的类型，是不是HTTP 相关，如果是，则转化为har格式 + var err error + switch ew.parser.ParserType() { + //case ParserTypeHttp2Request: + //case ParserTypeWebSocket: + //case ParserTypeHttpResponse: + //case ParserTypeHttpRequest: + //case ParserTypeHttp2Response: + //case ParserTypeNull: + // fallthrough + default: + err = ew.writeToChan(fmt.Sprintf("UUID:%s, Name:%s, Type:%d, Length:%d\n%s\n", ew.UUID, ew.parser.Name(), ew.parser.ParserType(), len(b), b)) + } //iWorker只负责写入，不应该打印。 - e := ew.writeToChan(fmt.Sprintf("UUID:%s, Name:%s, Type:%d, Length:%d\n%s\n", ew.UUID, ew.parser.Name(), ew.parser.ParserType(), len(b), b)) + //ew.parser.Reset() // 设定状态、重置包类型 ew.status = ProcessStateInit ew.packetType = PacketTypeNull - return e + return err } func (ew *eventWorker) writeEvent(e event.IEventStruct) { @@ -153,8 +166,6 @@ func (ew *eventWorker) Run() { // 输出包 if ew.tickerCount > MaxTickerCount { //ew.processor.GetLogger().Printf("eventWorker TickerCount > %d, event closed.", MaxTickerCount) - ew.processor.delWorkerByUUID(ew) - /* When returned from delWorkerByUUID(), there are two possibilities: 1) no routine can touch it. @@ -205,6 +216,9 @@ func (ew *eventWorker) Close() { ew.ticker.Stop() _ = ew.Display() ew.tickerCount = 0 + + // 关闭时清除 + ew.processor.delWorkerByUUID(ew) } func (ew *eventWorker) Get() { diff --git a/pkg/event_processor/processor.go b/pkg/event_processor/processor.go index 6c13300ec..99f599a3b 100644 --- a/pkg/event_processor/processor.go +++ b/pkg/event_processor/processor.go @@ -18,6 +18,7 @@ import ( "errors" "fmt" "github.com/gojue/ecapture/user/event" + "github.com/rs/zerolog" "io" "sync" ) @@ -37,12 +38,15 @@ type EventProcessor struct { // key为 PID+UID+COMMON等确定唯一的信息 workerQueue map[string]IWorker // log - logger io.Writer + logger *zerolog.Logger + eventLogger io.Writer closeChan chan bool // output model - isHex bool + isHex bool + appName string + appVersion string } func (ep *EventProcessor) GetLogger() io.Writer { @@ -68,7 +72,7 @@ func (ep *EventProcessor) Serve() error { return errors.Join(err, err1) } case s := <-ep.outComing: - _, _ = ep.GetLogger().Write([]byte(s)) + _, _ = ep.eventLogger.Write([]byte(s)) case _ = <-ep.closeChan: return nil } @@ -76,7 +80,7 @@ func (ep *EventProcessor) Serve() error { } func (ep *EventProcessor) dispatch(e event.IEventStruct) error { - //ep.logger.Printf("event ID:%s", e.GetUUID()) + ep.logger.Debug().Msgf("event ID:%s", e.GetUUID()) var uuid = e.GetUUID() found, eWorker := ep.getWorkerByUUID(uuid) if !found { @@ -150,12 +154,17 @@ func (ep *EventProcessor) Close() error { return nil } -func NewEventProcessor(logger io.Writer, isHex bool) *EventProcessor { +// NewEventProcessor 创建事件处理器 +func NewEventProcessor(logger *zerolog.Logger, eventLogger io.Writer, isHex bool, appName, appVer string) *EventProcessor { var ep *EventProcessor ep = &EventProcessor{} + // TODO 拆分为数据、日志两个通道 ep.logger = logger + ep.eventLogger = eventLogger ep.isHex = isHex ep.isClosed = false + ep.appName = appName + ep.appVersion = appVer ep.init() return ep } diff --git a/pkg/event_processor/processor_test.go b/pkg/event_processor/processor_test.go index 5380f4b41..afc5a2fbe 100644 --- a/pkg/event_processor/processor_test.go +++ b/pkg/event_processor/processor_test.go @@ -39,7 +39,7 @@ func TestEventProcessor_Serve(t *testing.T) { t.Fatal(e) } logger.SetOutput(f) - ep := NewEventProcessor(f, true) + ep := NewEventProcessor(f, true, "ecapture_test", "1.0.0") go func() { var err error err = ep.Serve() diff --git a/pkg/har/ctx.go b/pkg/har/ctx.go new file mode 100644 index 000000000..6c0663313 --- /dev/null +++ b/pkg/har/ctx.go @@ -0,0 +1,76 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package har collects HTTP requests and responses and stores them in HAR format. +// +// For more information on HAR, see: +// https://w3c.github.io/web-performance/specs/HAR/Overview.html + +// from https://github.com/google/martian + +package har + +import ( + "crypto/rand" + "encoding/hex" + "net/http" + "sync" +) + +var ( + ctxmu sync.RWMutex + ctxs = make(map[*http.Request]string) +) + +// NewContext returns a context for the in-flight HTTP request. +func NewContext(req *http.Request) string { + ctxmu.RLock() + defer ctxmu.RUnlock() + + return ctxs[req] +} + +// unlink removes the context for request. +func unlink(req *http.Request) { + ctxmu.Lock() + defer ctxmu.Unlock() + + delete(ctxs, req) +} + +func genID() (string, error) { + src := make([]byte, 8) + if _, err := rand.Read(src); err != nil { + return "", err + } + return hex.EncodeToString(src), nil +} + +func TestContext(req *http.Request) (func(), error) { + ctxmu.Lock() + defer ctxmu.Unlock() + + ctx, ok := ctxs[req] + if ok { + return func() { unlink(req) }, nil + } + var err error + ctx, err = genID() + if err != nil { + return nil, err + } + ctxs[req] = ctx + + return func() { unlink(req) }, nil +} diff --git a/pkg/har/har.go b/pkg/har/har.go new file mode 100644 index 000000000..83378c0b7 --- /dev/null +++ b/pkg/har/har.go @@ -0,0 +1,824 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package har collects HTTP requests and responses and stores them in HAR format. +// +// For more information on HAR, see: +// https://w3c.github.io/web-performance/specs/HAR/Overview.html + +// from https://github.com/google/martian + +package har + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "fmt" + "github.com/gojue/ecapture/pkg/messageview" + "github.com/gojue/ecapture/pkg/util/proxy" + "io" + "io/ioutil" + "mime" + "mime/multipart" + "net/http" + "net/url" + "strings" + "sync" + "time" + "unicode/utf8" +) + +// Logger maintains request and response log entries. +type Logger struct { + bodyLogging func(*http.Response) bool + postDataLogging func(*http.Request) bool + + creator *Creator + + mu sync.Mutex + entries map[string]*Entry + tail *Entry +} + +// HAR is the top level object of a HAR log. +type HAR struct { + Log *Log `json:"log"` +} + +// Log is the HAR HTTP request and response log. +type Log struct { + // Version number of the HAR format. + Version string `json:"version"` + // Creator holds information about the log creator application. + Creator *Creator `json:"creator"` + // Entries is a list containing requests and responses. + Entries []*Entry `json:"entries"` +} + +// Creator is the program responsible for generating the log. Martian, in this case. +type Creator struct { + // Name of the log creator application. + Name string `json:"name"` + // Version of the log creator application. + Version string `json:"version"` +} + +// Entry is a individual log entry for a request or response. +type Entry struct { + // ID is the unique ID for the entry. + ID string `json:"_id"` + // StartedDateTime is the date and time stamp of the request start (ISO 8601). + StartedDateTime time.Time `json:"startedDateTime"` + // Time is the total elapsed time of the request in milliseconds. + Time int64 `json:"time"` + // Request contains the detailed information about the request. + Request *Request `json:"request"` + // Response contains the detailed information about the response. + Response *Response `json:"response,omitempty"` + // Cache contains information about a request coming from browser cache. + Cache *Cache `json:"cache"` + // Timings describes various phases within request-response round trip. All + // times are specified in milliseconds. + Timings *Timings `json:"timings"` + next *Entry +} + +// Request holds data about an individual HTTP request. +type Request struct { + // Method is the request method (GET, POST, ...). + Method string `json:"method"` + // URL is the absolute URL of the request (fragments are not included). + URL string `json:"url"` + // HTTPVersion is the Request HTTP version (HTTP/1.1). + HTTPVersion string `json:"httpVersion"` + // Cookies is a list of cookies. + Cookies []Cookie `json:"cookies"` + // Headers is a list of headers. + Headers []Header `json:"headers"` + // QueryString is a list of query parameters. + QueryString []QueryString `json:"queryString"` + // PostData is the posted data information. + PostData *PostData `json:"postData,omitempty"` + // HeaderSize is the Total number of bytes from the start of the HTTP request + // message until (and including) the double CLRF before the body. Set to -1 + // if the info is not available. + HeadersSize int64 `json:"headersSize"` + // BodySize is the size of the request body (POST data payload) in bytes. Set + // to -1 if the info is not available. + BodySize int64 `json:"bodySize"` +} + +// Response holds data about an individual HTTP response. +type Response struct { + // Status is the response status code. + Status int `json:"status"` + // StatusText is the response status description. + StatusText string `json:"statusText"` + // HTTPVersion is the Response HTTP version (HTTP/1.1). + HTTPVersion string `json:"httpVersion"` + // Cookies is a list of cookies. + Cookies []Cookie `json:"cookies"` + // Headers is a list of headers. + Headers []Header `json:"headers"` + // Content contains the details of the response body. + Content *Content `json:"content"` + // RedirectURL is the target URL from the Location response header. + RedirectURL string `json:"redirectURL"` + // HeadersSize is the total number of bytes from the start of the HTTP + // request message until (and including) the double CLRF before the body. + // Set to -1 if the info is not available. + HeadersSize int64 `json:"headersSize"` + // BodySize is the size of the request body (POST data payload) in bytes. Set + // to -1 if the info is not available. + BodySize int64 `json:"bodySize"` +} + +// Cache contains information about a request coming from browser cache. +type Cache struct { + // Has no fields as they are not supported, but HAR requires the "cache" + // object to exist. +} + +// Timings describes various phases within request-response round trip. All +// times are specified in milliseconds +type Timings struct { + // Send is the time required to send HTTP request to the server. + Send int64 `json:"send"` + // Wait is the time spent waiting for a response from the server. + Wait int64 `json:"wait"` + // Receive is the time required to read entire response from server or cache. + Receive int64 `json:"receive"` +} + +// Cookie is the data about a cookie on a request or response. +type Cookie struct { + // Name is the cookie name. + Name string `json:"name"` + // Value is the cookie value. + Value string `json:"value"` + // Path is the path pertaining to the cookie. + Path string `json:"path,omitempty"` + // Domain is the host of the cookie. + Domain string `json:"domain,omitempty"` + // Expires contains cookie expiration time. + Expires time.Time `json:"-"` + // Expires8601 contains cookie expiration time in ISO 8601 format. + Expires8601 string `json:"expires,omitempty"` + // HTTPOnly is set to true if the cookie is HTTP only, false otherwise. + HTTPOnly bool `json:"httpOnly,omitempty"` + // Secure is set to true if the cookie was transmitted over SSL, false + // otherwise. + Secure bool `json:"secure,omitempty"` +} + +// Header is an HTTP request or response header. +type Header struct { + // Name is the header name. + Name string `json:"name"` + // Value is the header value. + Value string `json:"value"` +} + +// QueryString is a query string parameter on a request. +type QueryString struct { + // Name is the query parameter name. + Name string `json:"name"` + // Value is the query parameter value. + Value string `json:"value"` +} + +// PostData describes posted data on a request. +type PostData struct { + // MimeType is the MIME type of the posted data. + MimeType string `json:"mimeType"` + // Params is a list of posted parameters (in case of URL encoded parameters). + Params []Param `json:"params"` + // Text contains the posted data. Although its type is string, it may contain + // binary data. + Text string `json:"text"` +} + +// pdBinary is the JSON representation of binary PostData. +type pdBinary struct { + MimeType string `json:"mimeType"` + // Params is a list of posted parameters (in case of URL encoded parameters). + Params []Param `json:"params"` + Text []byte `json:"text"` + Encoding string `json:"encoding"` +} + +// MarshalJSON returns a JSON representation of binary PostData. +func (p *PostData) MarshalJSON() ([]byte, error) { + if utf8.ValidString(p.Text) { + type noMethod PostData // avoid infinite recursion + return json.Marshal((*noMethod)(p)) + } + return json.Marshal(pdBinary{ + MimeType: p.MimeType, + Params: p.Params, + Text: []byte(p.Text), + Encoding: "base64", + }) +} + +// UnmarshalJSON populates PostData based on the []byte representation of +// the binary PostData. +func (p *PostData) UnmarshalJSON(data []byte) error { + if bytes.Equal(data, []byte("null")) { // conform to json.Unmarshaler spec + return nil + } + var enc struct { + Encoding string `json:"encoding"` + } + if err := json.Unmarshal(data, &enc); err != nil { + return err + } + if enc.Encoding != "base64" { + type noMethod PostData // avoid infinite recursion + return json.Unmarshal(data, (*noMethod)(p)) + } + var pb pdBinary + if err := json.Unmarshal(data, &pb); err != nil { + return err + } + p.MimeType = pb.MimeType + p.Params = pb.Params + p.Text = string(pb.Text) + return nil +} + +// Param describes an individual posted parameter. +type Param struct { + // Name of the posted parameter. + Name string `json:"name"` + // Value of the posted parameter. + Value string `json:"value,omitempty"` + // Filename of a posted file. + Filename string `json:"fileName,omitempty"` + // ContentType is the content type of a posted file. + ContentType string `json:"contentType,omitempty"` +} + +// Content describes details about response content. +type Content struct { + // Size is the length of the returned content in bytes. Should be equal to + // response.bodySize if there is no compression and bigger when the content + // has been compressed. + Size int64 `json:"size"` + // MimeType is the MIME type of the response text (value of the Content-Type + // response header). + MimeType string `json:"mimeType"` + // Text contains the response body sent from the server or loaded from the + // browser cache. This field is populated with fully decoded version of the + // respose body. + Text []byte `json:"text,omitempty"` + // The desired encoding to use for the text field when encoding to JSON. + Encoding string `json:"encoding,omitempty"` +} + +// For marshaling Content to and from json. This works around the json library's +// default conversion of []byte to base64 encoded string. +type contentJSON struct { + Size int64 `json:"size"` + MimeType string `json:"mimeType"` + + // Text contains the response body sent from the server or loaded from the + // browser cache. This field is populated with textual content only. The text + // field is either HTTP decoded text or a encoded (e.g. "base64") + // representation of the response body. Leave out this field if the + // information is not available. + Text string `json:"text,omitempty"` + + // Encoding used for response text field e.g "base64". Leave out this field + // if the text field is HTTP decoded (decompressed & unchunked), than + // trans-coded from its original character set into UTF-8. + Encoding string `json:"encoding,omitempty"` +} + +// MarshalJSON marshals the byte slice into json after encoding based on c.Encoding. +func (c Content) MarshalJSON() ([]byte, error) { + var txt string + switch c.Encoding { + case "base64": + txt = base64.StdEncoding.EncodeToString(c.Text) + case "": + txt = string(c.Text) + default: + return nil, fmt.Errorf("unsupported encoding for Content.Text: %s", c.Encoding) + } + + cj := contentJSON{ + Size: c.Size, + MimeType: c.MimeType, + Text: txt, + Encoding: c.Encoding, + } + return json.Marshal(cj) +} + +// UnmarshalJSON unmarshals the bytes slice into Content. +func (c *Content) UnmarshalJSON(data []byte) error { + var cj contentJSON + if err := json.Unmarshal(data, &cj); err != nil { + return err + } + + var txt []byte + var err error + switch cj.Encoding { + case "base64": + txt, err = base64.StdEncoding.DecodeString(cj.Text) + if err != nil { + return fmt.Errorf("failed to decode base64-encoded Content.Text: %v", err) + } + case "": + txt = []byte(cj.Text) + default: + return fmt.Errorf("unsupported encoding for Content.Text: %s", cj.Encoding) + } + + c.Size = cj.Size + c.MimeType = cj.MimeType + c.Text = txt + c.Encoding = cj.Encoding + return nil +} + +// Option is a configurable setting for the logger. +type Option func(l *Logger) + +// PostDataLogging returns an option that configures request post data logging. +func PostDataLogging(enabled bool) Option { + return func(l *Logger) { + l.postDataLogging = func(*http.Request) bool { + return enabled + } + } +} + +// PostDataLoggingForContentTypes returns an option that logs request bodies based +// on opting in to the Content-Type of the request. +func PostDataLoggingForContentTypes(cts ...string) Option { + return func(l *Logger) { + l.postDataLogging = func(req *http.Request) bool { + rct := req.Header.Get("Content-Type") + + for _, ct := range cts { + if strings.HasPrefix(strings.ToLower(rct), strings.ToLower(ct)) { + return true + } + } + + return false + } + } +} + +// SkipPostDataLoggingForContentTypes returns an option that logs request bodies based +// on opting out of the Content-Type of the request. +func SkipPostDataLoggingForContentTypes(cts ...string) Option { + return func(l *Logger) { + l.postDataLogging = func(req *http.Request) bool { + rct := req.Header.Get("Content-Type") + + for _, ct := range cts { + if strings.HasPrefix(strings.ToLower(rct), strings.ToLower(ct)) { + return false + } + } + + return true + } + } +} + +// BodyLogging returns an option that configures response body logging. +func BodyLogging(enabled bool) Option { + return func(l *Logger) { + l.bodyLogging = func(*http.Response) bool { + return enabled + } + } +} + +// BodyLoggingForContentTypes returns an option that logs response bodies based +// on opting in to the Content-Type of the response. +func BodyLoggingForContentTypes(cts ...string) Option { + return func(l *Logger) { + l.bodyLogging = func(res *http.Response) bool { + rct := res.Header.Get("Content-Type") + + for _, ct := range cts { + if strings.HasPrefix(strings.ToLower(rct), strings.ToLower(ct)) { + return true + } + } + + return false + } + } +} + +// SkipBodyLoggingForContentTypes returns an option that logs response bodies based +// on opting out of the Content-Type of the response. +func SkipBodyLoggingForContentTypes(cts ...string) Option { + return func(l *Logger) { + l.bodyLogging = func(res *http.Response) bool { + rct := res.Header.Get("Content-Type") + + for _, ct := range cts { + if strings.HasPrefix(strings.ToLower(rct), strings.ToLower(ct)) { + return false + } + } + + return true + } + } +} + +// NewLogger returns a HAR logger. The returned +// logger logs all request post data and response bodies by default. +func NewLogger(appName, appVer string) *Logger { + l := &Logger{ + creator: &Creator{ + Name: appName, + Version: appVer, + }, + entries: make(map[string]*Entry), + } + l.SetOption(BodyLogging(true)) + l.SetOption(PostDataLogging(true)) + return l +} + +// SetOption sets configurable options on the logger. +func (l *Logger) SetOption(opts ...Option) { + for _, opt := range opts { + opt(l) + } +} + +// ModifyRequest logs requests. +func (l *Logger) ModifyRequest(req *http.Request) error { + id, err := genID() + if err != nil { + return err + } + return l.RecordRequest(id, req) +} + +// RecordRequest logs the HTTP request with the given ID. The ID should be unique +// per request/response pair. +func (l *Logger) RecordRequest(id string, req *http.Request) error { + hreq, err := NewRequest(req, l.postDataLogging(req)) + if err != nil { + return err + } + + entry := &Entry{ + ID: id, + StartedDateTime: time.Now().UTC(), + Request: hreq, + Cache: &Cache{}, + Timings: &Timings{}, + } + + l.mu.Lock() + defer l.mu.Unlock() + + if _, exists := l.entries[id]; exists { + return fmt.Errorf("Duplicate request ID: %s", id) + } + l.entries[id] = entry + if l.tail == nil { + l.tail = entry + } + entry.next = l.tail.next + l.tail.next = entry + l.tail = entry + + return nil +} + +// NewRequest constructs and returns a Request from req. If withBody is true, +// req.Body is read to EOF and replaced with a copy in a bytes.Buffer. An error +// is returned (and req.Body may be in an intermediate state) if an error is +// returned from req.Body.Read. +func NewRequest(req *http.Request, withBody bool) (*Request, error) { + r := &Request{ + Method: req.Method, + URL: req.URL.String(), + HTTPVersion: req.Proto, + HeadersSize: -1, + BodySize: req.ContentLength, + QueryString: []QueryString{}, + Headers: headers(proxy.RequestHeader(req).Map()), + Cookies: cookies(req.Cookies()), + } + + for n, vs := range req.URL.Query() { + for _, v := range vs { + r.QueryString = append(r.QueryString, QueryString{ + Name: n, + Value: v, + }) + } + } + + pd, err := postData(req, withBody) + if err != nil { + return nil, err + } + r.PostData = pd + + return r, nil +} + +// ModifyResponse logs responses. +func (l *Logger) ModifyResponse(res *http.Response) error { + id, err := genID() + if err != nil { + return err + } + return l.RecordResponse(id, res) +} + +// RecordResponse logs an HTTP response, associating it with the previously-logged +// HTTP request with the same ID. +func (l *Logger) RecordResponse(id string, res *http.Response) error { + hres, err := NewResponse(res, l.bodyLogging(res)) + if err != nil { + return err + } + + l.mu.Lock() + defer l.mu.Unlock() + + if e, ok := l.entries[id]; ok { + e.Response = hres + e.Time = time.Since(e.StartedDateTime).Nanoseconds() / 1000000 + } + + return nil +} + +// NewResponse constructs and returns a Response from resp. If withBody is true, +// resp.Body is read to EOF and replaced with a copy in a bytes.Buffer. An error +// is returned (and resp.Body may be in an intermediate state) if an error is +// returned from resp.Body.Read. +func NewResponse(res *http.Response, withBody bool) (*Response, error) { + r := &Response{ + HTTPVersion: res.Proto, + Status: res.StatusCode, + StatusText: http.StatusText(res.StatusCode), + HeadersSize: -1, + BodySize: res.ContentLength, + Headers: headers(proxy.ResponseHeader(res).Map()), + Cookies: cookies(res.Cookies()), + } + + if res.StatusCode >= 300 && res.StatusCode < 400 { + r.RedirectURL = res.Header.Get("Location") + } + + r.Content = &Content{ + Encoding: "base64", + MimeType: res.Header.Get("Content-Type"), + } + + if withBody { + mv := messageview.New() + if err := mv.SnapshotResponse(res); err != nil { + return nil, err + } + + br, err := mv.BodyReader(messageview.Decode()) + if err != nil { + return nil, err + } + + body, err := ioutil.ReadAll(br) + if err != nil { + return nil, err + } + + r.Content.Text = body + r.Content.Size = int64(len(body)) + } + return r, nil +} + +// Export returns the in-memory log. +func (l *Logger) Export() *HAR { + l.mu.Lock() + defer l.mu.Unlock() + + es := make([]*Entry, 0, len(l.entries)) + curr := l.tail + for curr != nil { + curr = curr.next + es = append(es, curr) + if curr == l.tail { + break + } + } + + return l.makeHAR(es) +} + +// ExportAndReset returns the in-memory log for completed requests, clearing them. +func (l *Logger) ExportAndReset() *HAR { + l.mu.Lock() + defer l.mu.Unlock() + + es := make([]*Entry, 0, len(l.entries)) + curr := l.tail + prev := l.tail + var first *Entry + for curr != nil { + curr = curr.next + if curr.Response != nil { + es = append(es, curr) + delete(l.entries, curr.ID) + } else { + if first == nil { + first = curr + } + prev.next = curr + prev = curr + } + if curr == l.tail { + break + } + } + if len(l.entries) == 0 { + l.tail = nil + } else { + l.tail = prev + l.tail.next = first + } + + return l.makeHAR(es) +} + +func (l *Logger) makeHAR(es []*Entry) *HAR { + return &HAR{ + Log: &Log{ + Version: "1.2", + Creator: l.creator, + Entries: es, + }, + } +} + +// Reset clears the in-memory log of entries. +func (l *Logger) Reset() { + l.mu.Lock() + defer l.mu.Unlock() + + l.entries = make(map[string]*Entry) + l.tail = nil +} + +func cookies(cs []*http.Cookie) []Cookie { + hcs := make([]Cookie, 0, len(cs)) + + for _, c := range cs { + var expires string + if !c.Expires.IsZero() { + expires = c.Expires.Format(time.RFC3339) + } + + hcs = append(hcs, Cookie{ + Name: c.Name, + Value: c.Value, + Path: c.Path, + Domain: c.Domain, + HTTPOnly: c.HttpOnly, + Secure: c.Secure, + Expires: c.Expires, + Expires8601: expires, + }) + } + + return hcs +} + +func headers(hs http.Header) []Header { + hhs := make([]Header, 0, len(hs)) + + for n, vs := range hs { + for _, v := range vs { + hhs = append(hhs, Header{ + Name: n, + Value: v, + }) + } + } + + return hhs +} + +func postData(req *http.Request, logBody bool) (*PostData, error) { + // If the request has no body (no Content-Length and Transfer-Encoding isn't + // chunked), skip the post data. + if req.ContentLength <= 0 && len(req.TransferEncoding) == 0 { + return nil, nil + } + + ct := req.Header.Get("Content-Type") + mt, ps, err := mime.ParseMediaType(ct) + if err != nil { + // print log ? + //log.Errorf("har: cannot parse Content-Type header %q: %v", ct, err) + mt = ct + } + + pd := &PostData{ + MimeType: mt, + Params: []Param{}, + } + + if !logBody { + return pd, nil + } + + mv := messageview.New() + if err := mv.SnapshotRequest(req); err != nil { + return nil, err + } + + br, err := mv.BodyReader() + if err != nil { + return nil, err + } + + switch mt { + case "multipart/form-data": + mpr := multipart.NewReader(br, ps["boundary"]) + + for { + p, err := mpr.NextPart() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + defer p.Close() + + body, err := ioutil.ReadAll(p) + if err != nil { + return nil, err + } + + pd.Params = append(pd.Params, Param{ + Name: p.FormName(), + Filename: p.FileName(), + ContentType: p.Header.Get("Content-Type"), + Value: string(body), + }) + } + case "application/x-www-form-urlencoded": + body, err := io.ReadAll(br) + if err != nil { + return nil, err + } + + vs, err := url.ParseQuery(string(body)) + if err != nil { + return nil, err + } + + for n, vs := range vs { + for _, v := range vs { + pd.Params = append(pd.Params, Param{ + Name: n, + Value: v, + }) + } + } + default: + body, err := io.ReadAll(br) + if err != nil { + return nil, err + } + + pd.Text = string(body) + } + + return pd, nil +} diff --git a/pkg/har/har_handlers.go b/pkg/har/har_handlers.go new file mode 100644 index 000000000..91f9e17eb --- /dev/null +++ b/pkg/har/har_handlers.go @@ -0,0 +1,99 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package har + +import ( + "encoding/json" + "log" + "net/http" + "net/url" + "strconv" +) + +type exportHandler struct { + logger *Logger +} + +type resetHandler struct { + logger *Logger +} + +// NewExportHandler returns an http.Handler for requesting HAR logs. +func NewExportHandler(l *Logger) http.Handler { + return &exportHandler{ + logger: l, + } +} + +// NewResetHandler returns an http.Handler for clearing in-memory log entries. +func NewResetHandler(l *Logger) http.Handler { + return &resetHandler{ + logger: l, + } +} + +// ServeHTTP writes the log in HAR format to the response body. +func (h *exportHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { + if req.Method != "GET" { + rw.Header().Add("Allow", "GET") + rw.WriteHeader(http.StatusMethodNotAllowed) + log.Printf("har.ServeHTTP: method not allowed: %s", req.Method) + return + } + log.Printf("exportHandler.ServeHTTP: writing HAR logs to ResponseWriter") + rw.Header().Set("Content-Type", "application/json; charset=utf-8") + + hl := h.logger.Export() + json.NewEncoder(rw).Encode(hl) +} + +// ServeHTTP resets the log, which clears its entries. +func (h *resetHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { + if !(req.Method == "POST" || req.Method == "DELETE") { + rw.Header().Add("Allow", "POST") + rw.Header().Add("Allow", "DELETE") + rw.WriteHeader(http.StatusMethodNotAllowed) + log.Printf("har: method not allowed: %s", req.Method) + return + } + + v, err := parseBoolQueryParam(req.URL.Query(), "return") + if err != nil { + log.Printf("har: invalid value for return param: %s", err) + rw.WriteHeader(http.StatusBadRequest) + return + } + + if v { + rw.Header().Set("Content-Type", "application/json; charset=utf-8") + hl := h.logger.ExportAndReset() + json.NewEncoder(rw).Encode(hl) + } else { + h.logger.Reset() + rw.WriteHeader(http.StatusNoContent) + } + log.Printf("resetHandler.ServeHTTP: HAR logs cleared") +} + +func parseBoolQueryParam(params url.Values, name string) (bool, error) { + if params[name] == nil { + return false, nil + } + v, err := strconv.ParseBool(params.Get("return")) + if err != nil { + return false, err + } + return v, nil +} diff --git a/pkg/har/har_handlers_test.go b/pkg/har/har_handlers_test.go new file mode 100644 index 000000000..30690d9e2 --- /dev/null +++ b/pkg/har/har_handlers_test.go @@ -0,0 +1,141 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package har + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" +) + +func TestExportHandlerServeHTTP(t *testing.T) { + logger := NewLogger("ecapture", "0.8.4") + + req, err := http.NewRequest("GET", "http://www.baidu.com", nil) + if err != nil { + t.Fatalf("http.NewRequest(): got %v, want no error", err) + } + + //if err := logger.ModifyRequest(req); err != nil { + // t.Fatalf("ModifyRequest(): got %v, want no error", err) + //} + // + //res := proxy.NewResponse(200, nil, req) + //if err := logger.ModifyResponse(res); err != nil { + // t.Fatalf("ModifyResponse(): got %v, want no error", err) + //} + + h := NewExportHandler(logger) + + req, err = http.NewRequest("GET", "/", nil) + if err != nil { + t.Fatalf("http.NewRequest(): got %v, want no error", err) + } + + rw := httptest.NewRecorder() + h.ServeHTTP(rw, req) + if got, want := rw.Code, http.StatusOK; got != want { + t.Errorf("rw.Code: got %d, want %d", got, want) + } + + hl := &HAR{} + if err := json.Unmarshal(rw.Body.Bytes(), hl); err != nil { + t.Fatalf("json.Unmarshal(): got %v, want no error", err) + } + + if got, want := len(hl.Log.Entries), 1; got != want { + t.Fatalf("len(hl.Log.Entries): got %v, want %v", got, want) + } + + entry := hl.Log.Entries[0] + if got, want := entry.Request.URL, "http://example.com"; got != want { + t.Errorf("Request.URL: got %q, want %q", got, want) + } + if got, want := entry.Response.Status, 200; got != want { + t.Errorf("Response.Status: got %d, want %d", got, want) + } + + rh := NewResetHandler(logger) + req, err = http.NewRequest("DELETE", "/", nil) + if err != nil { + t.Fatalf("http.NewRequest(): got %v, want no error", err) + } + + rw = httptest.NewRecorder() + rh.ServeHTTP(rw, req) + + req, err = http.NewRequest("GET", "/", nil) + if err != nil { + t.Fatalf("http.NewRequest(): got %v, want no error", err) + } + + rw = httptest.NewRecorder() + h.ServeHTTP(rw, req) + if got, want := rw.Code, http.StatusOK; got != want { + t.Errorf("rw.Code: got %d, want %d", got, want) + } + + hl = &HAR{} + if err := json.Unmarshal(rw.Body.Bytes(), hl); err != nil { + t.Fatalf("json.Unmarshal(): got %v, want no error", err) + } + + if got, want := len(hl.Log.Entries), 0; got != want { + t.Errorf("len(Log.Entries): got %v, want %v", got, want) + } + + req, err = http.NewRequest("DELETE", "/?return=1", nil) + if err != nil { + t.Fatalf("http.NewRequest(): got %v, want no error", err) + } + + rw = httptest.NewRecorder() + rh.ServeHTTP(rw, req) + if got, want := rw.Code, http.StatusOK; got != want { + t.Errorf("rw.Code: got %d, want %d", got, want) + } + + hl = &HAR{} + if err := json.Unmarshal(rw.Body.Bytes(), hl); err != nil { + t.Fatalf("json.Unmarshal(): got %v, want no error", err) + } + + if got, want := len(hl.Log.Entries), 0; got != want { + t.Errorf("len(Log.Entries): got %v, want %v", got, want) + } + + req, err = http.NewRequest("DELETE", "/?return=0", nil) + if err != nil { + t.Fatalf("http.NewRequest(): got %v, want no error", err) + } + + rw = httptest.NewRecorder() + rh.ServeHTTP(rw, req) + if got, want := rw.Code, http.StatusNoContent; got != want { + t.Errorf("rw.Code: got %d, want %d", got, want) + } + + req, err = http.NewRequest("DELETE", "/?return=notboolean", nil) + if err != nil { + t.Fatalf("http.NewRequest(): got %v, want no error", err) + } + + rw = httptest.NewRecorder() + rh.ServeHTTP(rw, req) + if got, want := rw.Code, http.StatusBadRequest; got != want { + t.Errorf("rw.Code: got %d, want %d", got, want) + } +} diff --git a/pkg/har/har_test.go b/pkg/har/har_test.go new file mode 100644 index 000000000..84b968198 --- /dev/null +++ b/pkg/har/har_test.go @@ -0,0 +1,954 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package har + +import ( + "bytes" + "encoding/json" + "fmt" + "github.com/gojue/ecapture/pkg/util/proxy" + "mime/multipart" + "net/http" + "reflect" + "strings" + "testing" + "time" +) + +func TestModifyRequest(t *testing.T) { + req, err := http.NewRequest("GET", "http://example.com/path?query=true", nil) + if err != nil { + t.Fatalf("http.NewRequest(): got %v, want no error", err) + } + req.Header.Add("Request-Header", "first") + req.Header.Add("Request-Header", "second") + + cookie := &http.Cookie{ + Name: "request", + Value: "cookie", + } + req.AddCookie(cookie) + + remove, err := TestContext(req) + if err != nil { + t.Fatalf("TestContext(): got %v, want no error", err) + } + defer remove() + + logger := NewLogger("ecapture", "0.8.4") + if err := logger.ModifyRequest(req); err != nil { + t.Fatalf("ModifyRequest(): got %v, want no error", err) + } + + log := logger.Export().Log + if got, want := log.Version, "1.2"; got != want { + t.Errorf("log.Version: got %q, want %q", got, want) + } + + if got, want := len(log.Entries), 1; got != want { + t.Fatalf("len(log.Entries): got %d, want %d", got, want) + } + + entry := log.Entries[0] + if got, want := time.Since(entry.StartedDateTime), time.Second; got > want { + t.Errorf("entry.StartedDateTime: got %s, want less than %s", got, want) + } + + hreq := entry.Request + if got, want := hreq.Method, "GET"; got != want { + t.Errorf("hreq.Method: got %q, want %q", got, want) + } + + if got, want := hreq.URL, "http://example.com/path?query=true"; got != want { + t.Errorf("hreq.URL: got %q, want %q", got, want) + } + + if got, want := hreq.HTTPVersion, "HTTP/1.1"; got != want { + t.Errorf("hreq.HTTPVersion: got %q, want %q", got, want) + } + + if got, want := hreq.BodySize, int64(0); got != want { + t.Errorf("hreq.BodySize: got %d, want %d", got, want) + } + + if got, want := hreq.HeadersSize, int64(-1); got != want { + t.Errorf("hreq.HeadersSize: got %d, want %d", got, want) + } + + if got, want := len(hreq.QueryString), 1; got != want { + t.Fatalf("len(hreq.QueryString): got %d, want %q", got, want) + } + + qs := hreq.QueryString[0] + if got, want := qs.Name, "query"; got != want { + t.Errorf("qs.Name: got %q, want %q", got, want) + } + if got, want := qs.Value, "true"; got != want { + t.Errorf("qs.Value: got %q, want %q", got, want) + } + + wantHeaders := http.Header{ + "Request-Header": {"first", "second"}, + "Cookie": {cookie.String()}, + "Host": {"example.com"}, + } + if got := headersToHTTP(hreq.Headers); !reflect.DeepEqual(got, wantHeaders) { + t.Errorf("headers:\ngot:\n%+v\nwant:\n%+v", got, wantHeaders) + } + + if got, want := len(hreq.Cookies), 1; got != want { + t.Fatalf("len(hreq.Cookies): got %d, want %d", got, want) + } + + hcookie := hreq.Cookies[0] + if got, want := hcookie.Name, "request"; got != want { + t.Errorf("hcookie.Name: got %q, want %q", got, want) + } + if got, want := hcookie.Value, "cookie"; got != want { + t.Errorf("hcookie.Value: got %q, want %q", got, want) + } +} + +func headersToHTTP(hs []Header) http.Header { + hh := http.Header{} + for _, h := range hs { + hh[h.Name] = append(hh[h.Name], h.Value) + } + return hh +} + +func TestModifyResponse(t *testing.T) { + req, err := http.NewRequest("GET", "http://example.com", nil) + if err != nil { + t.Fatalf("NewRequest(): got %v, want no error", err) + } + + remove, err := TestContext(req) + if err != nil { + t.Fatalf("TestContext(): got %v, want no error", err) + } + defer remove() + + res := proxy.NewResponse(301, strings.NewReader("response body"), req) + res.ContentLength = 13 + res.Header.Add("Response-Header", "first") + res.Header.Add("Response-Header", "second") + res.Header.Set("Location", "baidu.com") + + expires := time.Now() + cookie := &http.Cookie{ + Name: "response", + Value: "cookie", + Path: "/", + Domain: "example.com", + Expires: expires, + Secure: true, + HttpOnly: true, + } + res.Header.Set("Set-Cookie", cookie.String()) + + logger := NewLogger("ecapture", "0.8.4") + + if err := logger.ModifyRequest(req); err != nil { + t.Fatalf("ModifyRequest(): got %v, want no error", err) + } + + if err := logger.ModifyResponse(res); err != nil { + t.Fatalf("ModifyResponse(): got %v, want no error", err) + } + + log := logger.Export().Log + if got, want := len(log.Entries), 1; got != want { + t.Fatalf("len(log.Entries): got %d, want %d", got, want) + } + + fmt.Printf("log.Entries: %+v\n", log.Entries) + fmt.Printf("log.Entries[0].Response: %+v\n", log.Entries[0].Response) + hres := log.Entries[0].Response + if got, want := hres.Status, 301; got != want { + t.Errorf("hres.Status: got %d, want %d", got, want) + } + + if got, want := hres.StatusText, "Moved Permanently"; got != want { + t.Errorf("hres.StatusText: got %q, want %q", got, want) + } + + if got, want := hres.HTTPVersion, "HTTP/1.1"; got != want { + t.Errorf("hres.HTTPVersion: got %q, want %q", got, want) + } + + if got, want := hres.Content.Text, []byte("response body"); !bytes.Equal(got, want) { + t.Errorf("hres.Content.Text: got %q, want %q", got, want) + } + + wantHeaders := http.Header{ + "Response-Header": {"first", "second"}, + "Set-Cookie": {cookie.String()}, + "Location": {"google.com"}, + "Content-Length": {"13"}, + } + if got := headersToHTTP(hres.Headers); !reflect.DeepEqual(got, wantHeaders) { + t.Errorf("headers:\ngot:\n%+v\nwant:\n%+v", got, wantHeaders) + } + + if got, want := len(hres.Cookies), 1; got != want { + t.Fatalf("len(hres.Cookies): got %d, want %d", got, want) + } + + hcookie := hres.Cookies[0] + if got, want := hcookie.Name, "response"; got != want { + t.Errorf("hcookie.Name: got %q, want %q", got, want) + } + if got, want := hcookie.Value, "cookie"; got != want { + t.Errorf("hcookie.Value: got %q, want %q", got, want) + } + if got, want := hcookie.Path, "/"; got != want { + t.Errorf("hcookie.Path: got %q, want %q", got, want) + } + if got, want := hcookie.Domain, "example.com"; got != want { + t.Errorf("hcookie.Domain: got %q, want %q", got, want) + } + if got, want := hcookie.Expires, expires; got.Equal(want) { + t.Errorf("hcookie.Expires: got %s, want %s", got, want) + } + if !hcookie.HTTPOnly { + t.Error("hcookie.HTTPOnly: got false, want true") + } + if !hcookie.Secure { + t.Error("hcookie.Secure: got false, want true") + } +} + +func TestModifyRequestBodyURLEncoded(t *testing.T) { + logger := NewLogger("ecapture", "0.8.4") + + body := strings.NewReader("first=true&second=false") + req, err := http.NewRequest("POST", "http://example.com", body) + if err != nil { + t.Fatalf("http.NewRequest(): got %v, want no error", err) + } + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + + remove, err := TestContext(req) + if err != nil { + t.Fatalf("TestContext(): got %v, want no error", err) + } + defer remove() + + if err := logger.ModifyRequest(req); err != nil { + t.Fatalf("ModifyRequest(): got %v, want no error", err) + } + + log := logger.Export().Log + if got, want := len(log.Entries), 1; got != want { + t.Errorf("len(log.Entries): got %v, want %v", got, want) + } + + pd := log.Entries[0].Request.PostData + if got, want := pd.MimeType, "application/x-www-form-urlencoded"; got != want { + t.Errorf("PostData.MimeType: got %v, want %v", got, want) + } + + if got, want := len(pd.Params), 2; got != want { + t.Fatalf("len(PostData.Params): got %d, want %d", got, want) + } + + for _, p := range pd.Params { + var want string + switch p.Name { + case "first": + want = "true" + case "second": + want = "false" + default: + t.Errorf("PostData.Params: got %q, want to not be present", p.Name) + continue + } + + if got := p.Value; got != want { + t.Errorf("PostData.Params[%q]: got %q, want %q", p.Name, got, want) + } + } +} + +func TestModifyRequestBodyArbitraryContentType(t *testing.T) { + logger := NewLogger("ecapture", "0.8.4") + + body := "arbitrary binary data" + req, err := http.NewRequest("POST", "http://www.example.com", strings.NewReader(body)) + if err != nil { + t.Fatalf("http.NewRequest(): got %v, want no error", err) + } + + remove, err := TestContext(req) + if err != nil { + t.Fatalf("TestContext(): got %v, want no error", err) + } + defer remove() + + if err := logger.ModifyRequest(req); err != nil { + t.Fatalf("ModifyRequest(): got %v, want no error", err) + } + + log := logger.Export().Log + if got, want := len(log.Entries), 1; got != want { + t.Errorf("len(log.Entries): got %d, want %d", got, want) + } + + pd := log.Entries[0].Request.PostData + if got, want := pd.MimeType, ""; got != want { + t.Errorf("PostData.MimeType: got %q, want %q", got, want) + } + if got, want := len(pd.Params), 0; got != want { + t.Errorf("len(PostData.Params): got %d, want %d", got, want) + } + + if got, want := pd.Text, body; got != want { + t.Errorf("PostData.Text: got %q, want %q", got, want) + } +} + +func TestModifyRequestBodyMultipart(t *testing.T) { + logger := NewLogger("ecapture", "0.8.4") + + body := new(bytes.Buffer) + mpw := multipart.NewWriter(body) + mpw.SetBoundary("boundary") + + if err := mpw.WriteField("key", "value"); err != nil { + t.Errorf("mpw.WriteField(): got %v, want no error", err) + } + + w, err := mpw.CreateFormFile("file", "test.txt") + if _, err = w.Write([]byte("file contents")); err != nil { + t.Fatalf("Write(): got %v, want no error", err) + } + mpw.Close() + + req, err := http.NewRequest("POST", "http://example.com", body) + if err != nil { + t.Fatalf("http.NewRequest(): got %v, want no error", err) + } + req.Header.Set("Content-Type", mpw.FormDataContentType()) + + remove, err := TestContext(req) + if err != nil { + t.Fatalf("TestContext(): got %v, want no error", err) + } + defer remove() + + if err := logger.ModifyRequest(req); err != nil { + t.Fatalf("ModifyRequest(): got %v, want no error", err) + } + + log := logger.Export().Log + if got, want := len(log.Entries), 1; got != want { + t.Fatalf("len(log.Entries): got %d, want %d", got, want) + } + + pd := log.Entries[0].Request.PostData + if got, want := pd.MimeType, "multipart/form-data"; got != want { + t.Errorf("PostData.MimeType: got %q, want %q", got, want) + } + if got, want := len(pd.Params), 2; got != want { + t.Errorf("PostData.Params: got %d, want %d", got, want) + } + + for _, p := range pd.Params { + var want Param + + switch p.Name { + case "key": + want = Param{ + Filename: "", + ContentType: "", + Value: "value", + } + case "file": + want = Param{ + Filename: "test.txt", + ContentType: "application/octet-stream", + Value: "file contents", + } + default: + t.Errorf("pd.Params: got %q, want not to be present", p.Name) + continue + } + + if got, want := p.Filename, want.Filename; got != want { + t.Errorf("p.Filename: got %q, want %q", got, want) + } + if got, want := p.ContentType, want.ContentType; got != want { + t.Errorf("p.ContentType: got %q, want %q", got, want) + } + if got, want := p.Value, want.Value; got != want { + t.Errorf("p.Value: got %q, want %q", got, want) + } + } +} + +func TestModifyRequestErrorsOnDuplicateRequest(t *testing.T) { + logger := NewLogger("ecapture", "0.8.4") + + req, err := http.NewRequest("POST", "http://example.com", nil) + if err != nil { + t.Fatalf("http.NewRequest(): got %v, want no error", err) + } + + remove, err := TestContext(req) + if err != nil { + t.Fatalf("TestContext(): got %v, want no error", err) + } + defer remove() + + if err := logger.ModifyRequest(req); err != nil { + t.Fatalf("ModifyRequest(): got %v, want no error", err) + } + + if logger.ModifyRequest(req) == nil { + t.Fatalf("ModifyRequest(): was supposed to error") + } +} + +func TestHARExportsTime(t *testing.T) { + logger := NewLogger("ecapture", "0.8.4") + + req, err := http.NewRequest("GET", "http://example.com", nil) + if err != nil { + t.Fatalf("NewRequest(): got %v, want no error", err) + } + + remove, err := TestContext(req) + if err != nil { + t.Fatalf("TestContext(): got %v, want no error", err) + } + defer remove() + + if err := logger.ModifyRequest(req); err != nil { + t.Fatalf("ModifyRequest(): got %v, want no error", err) + } + + // Simulate fast network round trip. + time.Sleep(10 * time.Millisecond) + + res := proxy.NewResponse(200, nil, req) + + if err := logger.ModifyResponse(res); err != nil { + t.Fatalf("ModifyResponse(): got %v, want no error", err) + } + + log := logger.Export().Log + if got, want := len(log.Entries), 1; got != want { + t.Fatalf("len(log.Entries): got %v, want %v", got, want) + } + + entry := log.Entries[0] + min, max := int64(10), int64(100) + if got := entry.Time; got < min || got > max { + t.Errorf("entry.Time: got %dms, want between %dms and %vms", got, min, max) + } +} + +func TestReset(t *testing.T) { + logger := NewLogger("ecapture", "0.8.4") + + req, err := http.NewRequest("GET", "http://example.com", nil) + if err != nil { + t.Fatalf("NewRequest(): got %v, want no error", err) + } + + remove, err := TestContext(req) + if err != nil { + t.Fatalf("TestContext(): got %v, want no error", err) + } + defer remove() + + if err := logger.ModifyRequest(req); err != nil { + t.Fatalf("ModifyRequest(): got %v, want no error", err) + } + + log := logger.Export().Log + if got, want := len(log.Entries), 1; got != want { + t.Fatalf("len(log.Entries): got %d, want %d", got, want) + } + + logger.Reset() + + log = logger.Export().Log + if got, want := len(log.Entries), 0; got != want { + t.Errorf("len(log.Entries): got %d, want %d", got, want) + } +} + +func TestExportSortsEntries(t *testing.T) { + logger := NewLogger("ecapture", "0.8.4") + count := 10 + + for i := 0; i < count; i++ { + req, err := http.NewRequest("GET", "http://example.com", nil) + if err != nil { + t.Fatalf("NewRequest(): got %v, want no error", err) + } + + remove, err := TestContext(req) + if err != nil { + t.Fatalf("TestContext(): got %v, want no error", err) + } + defer remove() + + if err := logger.ModifyRequest(req); err != nil { + t.Fatalf("ModifyRequest(): got %v, want no error", err) + } + } + + log := logger.Export().Log + + for i := 0; i < count-1; i++ { + first := log.Entries[i] + second := log.Entries[i+1] + + if got, want := first.StartedDateTime, second.StartedDateTime; got.After(want) { + t.Errorf("entry.StartedDateTime: got %s, want to be before %s", got, want) + } + } +} + +func TestExportIgnoresOrphanedResponse(t *testing.T) { + logger := NewLogger("ecapture", "0.8.4") + + req, err := http.NewRequest("GET", "http://example.com", nil) + if err != nil { + t.Fatalf("http.NewRequest(): got %v, want no error", err) + } + + remove, err := TestContext(req) + if err != nil { + t.Fatalf("TestContext(): got %v, want no error", err) + } + defer remove() + + if err := logger.ModifyRequest(req); err != nil { + t.Fatalf("ModifyRequest(): got %v, want no error", err) + } + + // Reset before the response comes back. + logger.Reset() + + res := proxy.NewResponse(200, nil, req) + if err := logger.ModifyResponse(res); err != nil { + t.Fatalf("ModifyResponse(): got %v, want no error", err) + } + + log := logger.Export().Log + if got, want := len(log.Entries), 0; got != want { + t.Errorf("len(log.Entries): got %d, want %d", got, want) + } +} + +func TestExportAndResetResetsCompleteRequests(t *testing.T) { + logger := NewLogger("ecapture", "0.8.4") + + req, err := http.NewRequest("GET", "http://example.com", nil) + if err != nil { + t.Fatalf("http.NewRequest(): got %v, want no error", err) + } + + remove, err := TestContext(req) + if err != nil { + t.Fatalf("TestContext(): got %v, want no error", err) + } + defer remove() + + if err := logger.ModifyRequest(req); err != nil { + t.Fatalf("ModifyRequest(): got %v, want no error", err) + } + + res := proxy.NewResponse(200, nil, req) + if err := logger.ModifyResponse(res); err != nil { + t.Fatalf("ModifyResponse(): got %v, want no error", err) + } + + logger.ExportAndReset() + + log := logger.Export().Log + if got, want := len(log.Entries), 0; got != want { + t.Errorf("len(log.Entries): got %d, want %d", got, want) + } +} + +func TestExportAndResetLeavesPendingRequests(t *testing.T) { + logger := NewLogger("ecapture", "0.8.4") + + req, err := http.NewRequest("GET", "http://example.com", nil) + if err != nil { + t.Fatalf("http.NewRequest(): got %v, want no error", err) + } + + remove, err := TestContext(req) + if err != nil { + t.Fatalf("TestContext(): got %v, want no error", err) + } + defer remove() + + if err := logger.ModifyRequest(req); err != nil { + t.Fatalf("ModifyRequest(): got %v, want no error", err) + } + + logger.ExportAndReset() + + log := logger.Export().Log + if got, want := len(log.Entries), 1; got != want { + t.Errorf("len(log.Entries): got %d, want %d", got, want) + } +} + +func TestExportAndResetExportsCompleteRequests(t *testing.T) { + logger := NewLogger("ecapture", "0.8.4") + + req, err := http.NewRequest("GET", "http://example.com", nil) + if err != nil { + t.Fatalf("http.NewRequest(): got %v, want no error", err) + } + + remove, err := TestContext(req) + if err != nil { + t.Fatalf("TestContext(): got %v, want no error", err) + } + defer remove() + + if err := logger.ModifyRequest(req); err != nil { + t.Fatalf("ModifyRequest(): got %v, want no error", err) + } + + res := proxy.NewResponse(200, nil, req) + if err := logger.ModifyResponse(res); err != nil { + t.Fatalf("ModifyResponse(): got %v, want no error", err) + } + + log := logger.ExportAndReset().Log + if got, want := len(log.Entries), 1; got != want { + t.Errorf("len(log.Entries): got %d, want %d", got, want) + } +} + +func TestExportAndResetExportsCompleteRequestsWithPendingLeft(t *testing.T) { + logger := NewLogger("ecapture", "0.8.4") + + req, err := http.NewRequest("GET", "http://example.com", nil) + if err != nil { + t.Fatalf("http.NewRequest(): got %v, want no error", err) + } + + remove, err := TestContext(req) + if err != nil { + t.Fatalf("TestContext(): got %v, want no error", err) + } + defer remove() + + if err := logger.ModifyRequest(req); err != nil { + t.Fatalf("ModifyRequest(): got %v, want no error", err) + } + + req, err = http.NewRequest("GET", "http://example.com", nil) + if err != nil { + t.Fatalf("http.NewRequest(): got %v, want no error", err) + } + + remove, err = TestContext(req) + if err != nil { + t.Fatalf("TestContext(): got %v, want no error", err) + } + defer remove() + + if err := logger.ModifyRequest(req); err != nil { + t.Fatalf("ModifyRequest(): got %v, want no error", err) + } + + req, err = http.NewRequest("GET", "http://example.com", nil) + if err != nil { + t.Fatalf("http.NewRequest(): got %v, want no error", err) + } + + remove, err = TestContext(req) + if err != nil { + t.Fatalf("TestContext(): got %v, want no error", err) + } + defer remove() + + if err := logger.ModifyRequest(req); err != nil { + t.Fatalf("ModifyRequest(): got %v, want no error", err) + } + + res := proxy.NewResponse(200, nil, req) + if err := logger.ModifyResponse(res); err != nil { + t.Fatalf("ModifyResponse(): got %v, want no error", err) + } + + log := logger.ExportAndReset().Log + if got, want := len(log.Entries), 1; got != want { + t.Errorf("len(log.Entries): got %d, want %d", got, want) + } + + log = logger.Export().Log + if got, want := len(log.Entries), 2; got != want { + t.Errorf("len(log.Entries): got %d, want %d", got, want) + } +} + +func TestSkippingLogging(t *testing.T) { + req, err := http.NewRequest("GET", "http://example.com", nil) + if err != nil { + t.Fatalf("NewRequest(): got %v, want no error", err) + } + + remove, err := TestContext(req) + if err != nil { + t.Fatalf("TestContext(): got %v, want no error", err) + } + defer remove() + + logger := NewLogger("ecapture", "0.8.4") + + if err := logger.ModifyRequest(req); err != nil { + t.Fatalf("ModifyRequest(): got %v, want no error", err) + } + + res := proxy.NewResponse(200, nil, req) + if err := logger.ModifyResponse(res); err != nil { + t.Fatalf("ModifyResponse(): got %v, want no error", err) + } + + log := logger.Export().Log + if got, want := len(log.Entries), 0; got != want { + t.Fatalf("len(log.Entries): got %d, want %d", got, want) + } +} + +func TestOptionResponseBodyLogging(t *testing.T) { + req, err := http.NewRequest("GET", "http://example.com", nil) + if err != nil { + t.Fatalf("NewRequest(): got %v, want no error", err) + } + + remove, err := TestContext(req) + if err != nil { + t.Fatalf("TestContext(): got %v, want no error", err) + } + defer remove() + + bdr := strings.NewReader("{\"response\": \"body\"}") + res := proxy.NewResponse(200, bdr, req) + res.ContentLength = int64(bdr.Len()) + res.Header.Set("Content-Type", "application/json") + + logger := NewLogger("ecapture", "0.8.4") + + if err := logger.ModifyRequest(req); err != nil { + t.Fatalf("ModifyRequest(): got %v, want no error", err) + } + + if err := logger.ModifyResponse(res); err != nil { + t.Fatalf("ModifyResponse(): got %v, want no error", err) + } + + log := logger.Export().Log + if got, want := len(log.Entries), 1; got != want { + t.Fatalf("len(log.Entries): got %d, want %d", got, want) + } + + if got, want := string(log.Entries[0].Response.Content.Text), "{\"response\": \"body\"}"; got != want { + t.Fatalf("log.Entries[0].Response.Content.Text: got %s, want %s", got, want) + } + + logger = NewLogger("ecapture", "0.8.4") + logger.SetOption(BodyLogging(false)) + + if err := logger.ModifyRequest(req); err != nil { + t.Fatalf("ModifyRequest(): got %v, want no error", err) + } + + if err := logger.ModifyResponse(res); err != nil { + t.Fatalf("ModifyResponse(): got %v, want no error", err) + } + + log = logger.Export().Log + if got, want := len(log.Entries), 1; got != want { + t.Fatalf("len(log.Entries): got %d, want %d", got, want) + } + + if got, want := string(log.Entries[0].Response.Content.Text), ""; got != want { + t.Fatalf("log.Entries[0].Response.Content: got %s, want %s", got, want) + } + + logger = NewLogger("ecapture", "0.8.4") + logger.SetOption(BodyLoggingForContentTypes("application/json")) + + if err := logger.ModifyRequest(req); err != nil { + t.Fatalf("ModifyRequest(): got %v, want no error", err) + } + + if err := logger.ModifyResponse(res); err != nil { + t.Fatalf("ModifyResponse(): got %v, want no error", err) + } + + log = logger.Export().Log + if got, want := string(log.Entries[0].Response.Content.Text), "{\"response\": \"body\"}"; got != want { + t.Fatalf("log.Entries[0].Response.Content: got %s, want %s", got, want) + } + + logger = NewLogger("ecapture", "0.8.4") + logger.SetOption(SkipBodyLoggingForContentTypes("application/json")) + + if err := logger.ModifyRequest(req); err != nil { + t.Fatalf("ModifyRequest(): got %v, want no error", err) + } + + if err := logger.ModifyResponse(res); err != nil { + t.Fatalf("ModifyResponse(): got %v, want no error", err) + } + + log = logger.Export().Log + if got, want := string(log.Entries[0].Response.Content.Text), ""; got != want { + t.Fatalf("log.Entries[0].Response.Content: got %v, want %v", got, want) + } +} + +func TestOptionRequestPostDataLogging(t *testing.T) { + logger := NewLogger("ecapture", "0.8.4") + logger.SetOption(PostDataLoggingForContentTypes("application/x-www-form-urlencoded")) + + body := strings.NewReader("first=true&second=false") + req, err := http.NewRequest("POST", "http://example.com", body) + if err != nil { + t.Fatalf("http.NewRequest(): got %v, want no error", err) + } + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + + remove, err := TestContext(req) + if err != nil { + t.Fatalf("TestContext(): got %v, want no error", err) + } + defer remove() + + if err := logger.ModifyRequest(req); err != nil { + t.Fatalf("ModifyRequest(): got %v, want no error", err) + } + + log := logger.Export().Log + + for _, param := range log.Entries[0].Request.PostData.Params { + if param.Name == "first" { + if got, want := param.Value, "true"; got != want { + t.Fatalf("Params[%q].Value: got %s, want %s", param.Name, got, want) + } + } + + if param.Name == "second" { + if got, want := param.Value, "false"; got != want { + t.Fatalf("Params[%q].Value: got %s, want %s", param.Name, got, want) + } + } + } + + logger = NewLogger("ecapture", "0.8.4") + logger.SetOption(SkipPostDataLoggingForContentTypes("application/x-www-form-urlencoded")) + + body = strings.NewReader("first=true&second=false") + req, err = http.NewRequest("POST", "http://example.com", body) + if err != nil { + t.Fatalf("http.NewRequest(): got %v, want no error", err) + } + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + + remove, err = TestContext(req) + if err != nil { + t.Fatalf("TestContext(): got %v, want no error", err) + } + defer remove() + + if err := logger.ModifyRequest(req); err != nil { + t.Fatalf("ModifyRequest(): got %v, want no error", err) + } + + log = logger.Export().Log + if got, want := len(log.Entries[0].Request.PostData.Params), 0; got != want { + t.Fatalf("len(log.Entries[0].Request.PostData.Params): got %v, want %v", got, want) + } +} + +func TestJSONMarshalPostData(t *testing.T) { + // Verify that encoding/json round-trips har.PostData with both text and binary data. + for _, text := range []string{"hello", string([]byte{150, 151, 152})} { + want := &PostData{ + MimeType: "m", + Params: []Param{{Name: "n", Value: "v"}}, + Text: text, + } + data, err := json.Marshal(want) + if err != nil { + t.Fatal(err) + } + var got PostData + if err := json.Unmarshal(data, &got); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(&got, want) { + t.Errorf("got %+v, want %+v", &got, want) + } + } +} + +func TestJSONMarshalContent(t *testing.T) { + testCases := []struct { + name string + text []byte + encoding string + }{ + { + name: "binary data with base64 encoding", + text: []byte{120, 31, 99, 3}, + encoding: "base64", + }, + { + name: "ascii data with no encoding", + text: []byte("hello martian"), + }, + { + name: "ascii data with base64 encoding", + text: []byte("hello martian"), + encoding: "base64", + }, + } + + for _, c := range testCases { + want := Content{ + Size: int64(len(c.text)), + MimeType: "application/x-test", + Text: c.text, + Encoding: c.encoding, + } + data, err := json.Marshal(want) + if err != nil { + t.Fatal(err) + } + var got Content + if err := json.Unmarshal(data, &got); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, want) { + t.Errorf("got %+v, want %+v", got, want) + } + } +} diff --git a/user/config/iconfig.go b/user/config/iconfig.go index e890c612e..0f6b2706a 100644 --- a/user/config/iconfig.go +++ b/user/config/iconfig.go @@ -38,6 +38,10 @@ type IConfig interface { GetPerCpuMapSize() int SetPerCpuMapSize(int) EnableGlobalVar() bool // + AppName() string + SetAppName(string) + AppVersion() string + SetAppVersion(string) Bytes() []byte } @@ -68,6 +72,9 @@ type BaseConfig struct { LoggerAddr string `json:"logger_addr"` // logger address LoggerType uint8 `json:"logger_type"` // 0:stdout, 1:file, 2:tcp EventCollectorAddr string `json:"event_collector_addr"` // the server address that receives the captured event + + appName string // app name + appVersion string // app version } func (c *BaseConfig) GetPid() uint64 { @@ -148,3 +155,19 @@ func (c *BaseConfig) Bytes() []byte { } return b } + +func (c *BaseConfig) AppName() string { + return c.appName +} + +func (c *BaseConfig) SetAppName(name string) { + c.appName = name +} + +func (c *BaseConfig) AppVersion() string { + return c.appVersion +} + +func (c *BaseConfig) SetAppVersion(version string) { + c.appVersion = version +} diff --git a/user/module/imodule.go b/user/module/imodule.go index 874d7345c..0863342ad 100644 --- a/user/module/imodule.go +++ b/user/module/imodule.go @@ -100,8 +100,7 @@ func (m *Module) Init(ctx context.Context, logger *zerolog.Logger, conf config.I m.errChan = make(chan error) m.isKernelLess5_2 = false //set false default m.eventCollector = eventCollector - //var epl = epLogger{logger: logger} - m.processor = event_processor.NewEventProcessor(eventCollector, conf.GetHex()) + m.processor = event_processor.NewEventProcessor(logger, eventCollector, conf.GetHex(), conf.AppName(), conf.AppVersion()) kv, err := kernel.HostVersion() if err != nil { m.logger.Warn().Err(err).Msg("Unable to detect kernel version due to an error:%v.used non-Less5_2 bytecode.") @@ -189,7 +188,7 @@ func (m *Module) Name() string { } func (m *Module) Run() error { - m.logger.Info().Msg("Module.Run()") + m.logger.Debug().Msg("Module.Run()") // start err := m.child.Start() if err != nil { @@ -335,15 +334,15 @@ func (m *Module) ringbufEventReader(errChan chan error, em *ebpf.Map) { return } - var e event.IEventStruct - e, err = m.child.Decode(em, record.RawSample) + var ev event.IEventStruct + ev, err = m.child.Decode(em, record.RawSample) if err != nil { m.logger.Warn().Err(err).Msg("m.child.decode error") continue } // 上报数据 - m.Dispatcher(e) + m.Dispatcher(ev) } }() } @@ -405,7 +404,7 @@ func (m *Module) Dispatcher(e event.IEventStruct) { func (m *Module) Close() error { m.isClosed.Store(true) - m.logger.Info().Msg("iModule module close") + m.logger.Debug().Msg("iModule module close") for _, iClose := range m.reader { if err := iClose.Close(); err != nil { return err From 2b1c6edfd2e24127717dab41c07e266513c49266 Mon Sep 17 00:00:00 2001 From: CFC4N Date: Tue, 17 Sep 2024 22:24:26 +0800 Subject: [PATCH 2/8] pkg: Fixed the error that the handler parameter of the initialization event was missing Added makefile code for golang unit tests Signed-off-by: CFC4N --- Makefile | 22 ++++++++--- pkg/event_processor/processor_test.go | 55 +++++++++++++++++++++------ pkg/har/har_test.go | 10 ++++- 3 files changed, 67 insertions(+), 20 deletions(-) diff --git a/Makefile b/Makefile index cad5a630d..9b8aa4394 100644 --- a/Makefile +++ b/Makefile @@ -147,7 +147,7 @@ $(KERN_OBJECTS_NOCORE): %.nocore: %.c \ .checkver_$(CMD_GO) $(CMD_CLANG) \ $(EXTRA_CFLAGS_NOCORE) \ - $(BPFHEADER) \ + $(BPFHEADER) \ -I $(KERN_SRC_PATH)/arch/$(LINUX_ARCH)/include \ -I $(KERN_BUILD_PATH)/arch/$(LINUX_ARCH)/include/generated \ -I $(KERN_SRC_PATH)/include \ @@ -155,11 +155,11 @@ $(KERN_OBJECTS_NOCORE): %.nocore: %.c \ -I $(KERN_BUILD_PATH)/arch/$(LINUX_ARCH)/include/generated/uapi \ -I $(KERN_SRC_PATH)/include/uapi \ -I $(KERN_BUILD_PATH)/include/generated/uapi \ - -c $< \ - -o - |$(CMD_LLC) \ - -march=bpf \ - -filetype=obj \ - -o $(subst kern/,user/bytecode/,$(subst .c,_noncore.o,$<)) + -c $< \ + -o - |$(CMD_LLC) \ + -march=bpf \ + -filetype=obj \ + -o $(subst kern/,user/bytecode/,$(subst .c,_noncore.o,$<)) $(CMD_CLANG) \ $(EXTRA_CFLAGS_NOCORE) \ $(BPFHEADER) \ @@ -231,3 +231,13 @@ format: @clang-format -i -style=$(STYLE) kern/openssl_masterkey_3.2.h @clang-format -i -style=$(STYLE) kern/boringssl_masterkey.h @clang-format -i -style=$(STYLE) utils/*.c + + +# about string " -Wl,--no-gc-sections" in CGO_LDFLAGS, please see https://groups.google.com/g/golang-codereviews/c/ZnfJ5olFsnk. +# if without "-Wl,--no-gc-sections", the error message is "runtime/cgo(.text): relocation target stderr not defined" +.PHONY: gotest +gotest: + CGO_CFLAGS='-O2 -g -I$(CURDIR)/lib/libpcap/ -Wl,--no-gc-sections' \ + CGO_LDFLAGS='-O2 -g -L$(CURDIR)/lib/libpcap/ -lpcap -static' \ + GOOS=linux GOARCH=$(GOARCH) CC=$(CMD_CC_PREFIX)$(CMD_CC) \ + $(CMD_GO) test -v -race ./pkg/event_processor/... \ No newline at end of file diff --git a/pkg/event_processor/processor_test.go b/pkg/event_processor/processor_test.go index afc5a2fbe..ca09c0b11 100644 --- a/pkg/event_processor/processor_test.go +++ b/pkg/event_processor/processor_test.go @@ -3,14 +3,20 @@ package event_processor import ( "encoding/json" "fmt" - "io" - "log" + "github.com/rs/zerolog" "os" "strings" "testing" "time" ) +// ZeroLog print level +const ( + eTestEventLevel = zerolog.Level(88) + eTestEventName = "[DATA]" + eTestEventConsoleColor = 35 // colorMagenta +) + var ( testFile = "testdata/all.json" ) @@ -28,23 +34,48 @@ type SSLDataEventTmp struct { Data [4096]byte `json:"Data"` } +type eventTestWriter struct { + logger *zerolog.Logger +} + +func (e eventTestWriter) Write(p []byte) (n int, err error) { + e.logger.WithLevel(eTestEventLevel).Msgf("%s", p) + return len(p), nil +} + +func initTestLogger(stdoutFile string) (*zerolog.Logger, error) { + var logger zerolog.Logger + // append zerolog Global variables + zerolog.FormattedLevels[eTestEventLevel] = eTestEventName + zerolog.LevelColors[eTestEventLevel] = eTestEventConsoleColor + + consoleWriter := zerolog.ConsoleWriter{Out: os.Stdout, TimeFormat: time.RFC3339} + logger = zerolog.New(consoleWriter).With().Timestamp().Logger() + zerolog.SetGlobalLevel(zerolog.DebugLevel) + + f, err := os.Create(stdoutFile) + if err != nil { + return nil, err + } + multi := zerolog.MultiLevelWriter(consoleWriter, f) + logger = zerolog.New(multi).With().Timestamp().Logger() + return &logger, nil +} + func TestEventProcessor_Serve(t *testing.T) { - logger := log.Default() - //var buf bytes.Buffer - //logger.SetOutput(&buf) var output = "./output.log" - f, e := os.Create(output) - if e != nil { - t.Fatal(e) + lger, err := initTestLogger(output) + if err != nil { + t.Fatalf("init logger error: %s", err.Error()) } - logger.SetOutput(f) - ep := NewEventProcessor(f, true, "ecapture_test", "1.0.0") + var ecw = eventTestWriter{logger: lger} + + ep := NewEventProcessor(lger, ecw, true, "ecapture_test", "1.0.0") go func() { var err error err = ep.Serve() if err != nil { - //log.Fatalf(err.Error()) t.Error(err) return } @@ -78,7 +109,7 @@ func TestEventProcessor_Serve(t *testing.T) { <-tick.C err = ep.Close() - logger.SetOutput(io.Discard) + //logger.SetOutput(io.Discard) bufString, e := os.ReadFile(output) if e != nil { t.Fatal(e) diff --git a/pkg/har/har_test.go b/pkg/har/har_test.go index 84b968198..e42a3e275 100644 --- a/pkg/har/har_test.go +++ b/pkg/har/har_test.go @@ -152,7 +152,7 @@ func TestModifyResponse(t *testing.T) { Name: "response", Value: "cookie", Path: "/", - Domain: "example.com", + Domain: "baidu.com", Expires: expires, Secure: true, HttpOnly: true, @@ -177,6 +177,10 @@ func TestModifyResponse(t *testing.T) { fmt.Printf("log.Entries: %+v\n", log.Entries) fmt.Printf("log.Entries[0].Response: %+v\n", log.Entries[0].Response) hres := log.Entries[0].Response + //t.Logf("hres: %+v\n", hres) + if hres == nil { + t.Fatalf("log.Entries[0].Response: got nil, want not nil") + } if got, want := hres.Status, 301; got != want { t.Errorf("hres.Status: got %d, want %d", got, want) } @@ -767,7 +771,9 @@ func TestOptionResponseBodyLogging(t *testing.T) { if got, want := len(log.Entries), 1; got != want { t.Fatalf("len(log.Entries): got %d, want %d", got, want) } - + if log.Entries[0].Response == nil { + t.Fatalf("log.Entries[0].Response is nil") + } if got, want := string(log.Entries[0].Response.Content.Text), "{\"response\": \"body\"}"; got != want { t.Fatalf("log.Entries[0].Response.Content.Text: got %s, want %s", got, want) } From cc6ebf6cbb1db649b9e67ff312fa15433cd38f7f Mon Sep 17 00:00:00 2001 From: CFC4N Date: Tue, 17 Sep 2024 22:44:20 +0800 Subject: [PATCH 3/8] pkg: add package `messageview` (from google) Signed-off-by: CFC4N --- pkg/messageview/messageview.go | 289 +++++++++++ pkg/messageview/messageview_test.go | 757 ++++++++++++++++++++++++++++ 2 files changed, 1046 insertions(+) create mode 100644 pkg/messageview/messageview.go create mode 100644 pkg/messageview/messageview_test.go diff --git a/pkg/messageview/messageview.go b/pkg/messageview/messageview.go new file mode 100644 index 000000000..8d2e6e383 --- /dev/null +++ b/pkg/messageview/messageview.go @@ -0,0 +1,289 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package messageview provides no-op snapshots for HTTP requests and +// responses. +package messageview + +import ( + "bytes" + "compress/flate" + "compress/gzip" + "fmt" + "io" + "net/http" + "net/http/httputil" + "strings" +) + +// MessageView is a static view of an HTTP request or response. +type MessageView struct { + message []byte + cts []string + chunked bool + skipBody bool + compress string + bodyoffset int64 + traileroffset int64 +} + +type config struct { + decode bool +} + +// Option is a configuration option for a MessageView. +type Option func(*config) + +// Decode sets an option to decode the message body for logging purposes. +func Decode() Option { + return func(c *config) { + c.decode = true + } +} + +// New returns a new MessageView. +func New() *MessageView { + return &MessageView{} +} + +// SkipBody will skip reading the body when the view is loaded with a request +// or response. +func (mv *MessageView) SkipBody(skipBody bool) { + mv.skipBody = skipBody +} + +// SkipBodyUnlessContentType will skip reading the body unless the +// Content-Type matches one in cts. +func (mv *MessageView) SkipBodyUnlessContentType(cts ...string) { + mv.skipBody = true + mv.cts = cts +} + +// SnapshotRequest reads the request into the MessageView. If mv.skipBody is false +// it will also read the body into memory and replace the existing body with +// the in-memory copy. This method is semantically a no-op. +func (mv *MessageView) SnapshotRequest(req *http.Request) error { + buf := new(bytes.Buffer) + + fmt.Fprintf(buf, "%s %s HTTP/%d.%d\r\n", req.Method, + req.URL, req.ProtoMajor, req.ProtoMinor) + + if req.Host != "" { + fmt.Fprintf(buf, "Host: %s\r\n", req.Host) + } + + if tec := len(req.TransferEncoding); tec > 0 { + mv.chunked = req.TransferEncoding[tec-1] == "chunked" + fmt.Fprintf(buf, "Transfer-Encoding: %s\r\n", strings.Join(req.TransferEncoding, ", ")) + } + if !mv.chunked && req.ContentLength >= 0 { + fmt.Fprintf(buf, "Content-Length: %d\r\n", req.ContentLength) + } + + mv.compress = req.Header.Get("Content-Encoding") + + req.Header.WriteSubset(buf, map[string]bool{ + "Host": true, + "Content-Length": true, + "Transfer-Encoding": true, + }) + + fmt.Fprint(buf, "\r\n") + + mv.bodyoffset = int64(buf.Len()) + mv.traileroffset = int64(buf.Len()) + + ct := req.Header.Get("Content-Type") + if mv.skipBody && !mv.matchContentType(ct) || req.Body == nil { + mv.message = buf.Bytes() + return nil + } + + data, err := io.ReadAll(req.Body) + if err != nil { + return err + } + req.Body.Close() + + if mv.chunked { + cw := httputil.NewChunkedWriter(buf) + cw.Write(data) + cw.Close() + } else { + buf.Write(data) + } + + mv.traileroffset = int64(buf.Len()) + + req.Body = io.NopCloser(bytes.NewReader(data)) + + if req.Trailer != nil { + req.Trailer.Write(buf) + } else if mv.chunked { + fmt.Fprint(buf, "\r\n") + } + + mv.message = buf.Bytes() + + return nil +} + +// SnapshotResponse reads the response into the MessageView. If mv.headersOnly +// is false it will also read the body into memory and replace the existing +// body with the in-memory copy. This method is semantically a no-op. +func (mv *MessageView) SnapshotResponse(res *http.Response) error { + buf := new(bytes.Buffer) + + fmt.Fprintf(buf, "HTTP/%d.%d %s\r\n", res.ProtoMajor, res.ProtoMinor, res.Status) + + if tec := len(res.TransferEncoding); tec > 0 { + mv.chunked = res.TransferEncoding[tec-1] == "chunked" + fmt.Fprintf(buf, "Transfer-Encoding: %s\r\n", strings.Join(res.TransferEncoding, ", ")) + } + if !mv.chunked && res.ContentLength >= 0 { + fmt.Fprintf(buf, "Content-Length: %d\r\n", res.ContentLength) + } + + mv.compress = res.Header.Get("Content-Encoding") + // Do not uncompress if we have don't have the full contents. + if res.StatusCode == http.StatusNoContent || res.StatusCode == http.StatusPartialContent { + mv.compress = "" + } + + res.Header.WriteSubset(buf, map[string]bool{ + "Content-Length": true, + "Transfer-Encoding": true, + }) + + fmt.Fprint(buf, "\r\n") + + mv.bodyoffset = int64(buf.Len()) + mv.traileroffset = int64(buf.Len()) + + ct := res.Header.Get("Content-Type") + if mv.skipBody && !mv.matchContentType(ct) || res.Body == nil { + mv.message = buf.Bytes() + return nil + } + + data, err := io.ReadAll(res.Body) + if err != nil { + return err + } + res.Body.Close() + + if mv.chunked { + cw := httputil.NewChunkedWriter(buf) + cw.Write(data) + cw.Close() + } else { + buf.Write(data) + } + + mv.traileroffset = int64(buf.Len()) + + res.Body = io.NopCloser(bytes.NewReader(data)) + + if res.Trailer != nil { + res.Trailer.Write(buf) + } else if mv.chunked { + fmt.Fprint(buf, "\r\n") + } + + mv.message = buf.Bytes() + + return nil +} + +// Reader returns the an io.ReadCloser that reads the full HTTP message. +func (mv *MessageView) Reader(opts ...Option) (io.ReadCloser, error) { + hr := mv.HeaderReader() + br, err := mv.BodyReader(opts...) + if err != nil { + return nil, err + } + tr := mv.TrailerReader() + + return struct { + io.Reader + io.Closer + }{ + Reader: io.MultiReader(hr, br, tr), + Closer: br, + }, nil +} + +// HeaderReader returns an io.Reader that reads the HTTP Status-Line or +// HTTP Request-Line and headers. +func (mv *MessageView) HeaderReader() io.Reader { + r := bytes.NewReader(mv.message) + return io.NewSectionReader(r, 0, mv.bodyoffset) +} + +// BodyReader returns an io.ReadCloser that reads the HTTP request or response +// body. If mv.skipBody was set the reader will immediately return io.EOF. +// +// If the Decode option is passed the body will be unchunked if +// Transfer-Encoding is set to "chunked", and will decode the following +// Content-Encodings: gzip, deflate. +func (mv *MessageView) BodyReader(opts ...Option) (io.ReadCloser, error) { + var r io.Reader + + conf := &config{} + for _, o := range opts { + o(conf) + } + + br := bytes.NewReader(mv.message) + r = io.NewSectionReader(br, mv.bodyoffset, mv.traileroffset-mv.bodyoffset) + + if !conf.decode { + return io.NopCloser(r), nil + } + + if mv.chunked { + r = httputil.NewChunkedReader(r) + } + switch mv.compress { + case "gzip": + gr, err := gzip.NewReader(r) + if err != nil { + return nil, err + } + return gr, nil + case "deflate": + return flate.NewReader(r), nil + default: + return io.NopCloser(r), nil + } +} + +// TrailerReader returns an io.Reader that reads the HTTP request or response +// trailers, if present. +func (mv *MessageView) TrailerReader() io.Reader { + r := bytes.NewReader(mv.message) + end := int64(len(mv.message)) - mv.traileroffset + + return io.NewSectionReader(r, mv.traileroffset, end) +} + +func (mv *MessageView) matchContentType(mct string) bool { + for _, ct := range mv.cts { + if strings.HasPrefix(mct, ct) { + return true + } + } + + return false +} diff --git a/pkg/messageview/messageview_test.go b/pkg/messageview/messageview_test.go new file mode 100644 index 000000000..14b9f288a --- /dev/null +++ b/pkg/messageview/messageview_test.go @@ -0,0 +1,757 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package messageview + +import ( + "bufio" + "bytes" + "compress/flate" + "compress/gzip" + "github.com/gojue/ecapture/pkg/util/proxy" + "io" + "net/http" + "strings" + "testing" +) + +func TestRequestViewHeadersOnly(t *testing.T) { + body := strings.NewReader("body content") + req, err := http.NewRequest("GET", "http://example.com/path?k=v", body) + if err != nil { + t.Fatalf("http.NewRequest(): got %v, want no error", err) + } + req.ContentLength = int64(body.Len()) + req.Header.Set("Request-Header", "true") + + mv := New() + mv.SkipBody(true) + if err := mv.SnapshotRequest(req); err != nil { + t.Fatalf("SnapshotRequest(): got %v, want no error", err) + } + + got, err := io.ReadAll(mv.HeaderReader()) + if err != nil { + t.Fatalf("io.ReadAll(mv.HeaderReader()): got %v, want no error", err) + } + + hdrwant := "GET http://example.com/path?k=v HTTP/1.1\r\n" + + "Host: example.com\r\n" + + "Content-Length: 12\r\n" + + "Request-Header: true\r\n\r\n" + + if !bytes.Equal(got, []byte(hdrwant)) { + t.Fatalf("mv.HeaderReader(): got %q, want %q", got, hdrwant) + } + + br, err := mv.BodyReader() + if err != nil { + t.Fatalf("mv.BodyReader(): got %v, want no error", err) + } + + if _, err := br.Read(nil); err != io.EOF { + t.Fatalf("io.ReadAll(mv.BodyReader()): got %v, want io.EOF", err) + } + + r, err := mv.Reader() + if err != nil { + t.Fatalf("mv.Reader(): got %v, want no error", err) + } + got, err = io.ReadAll(r) + if err != nil { + t.Fatalf("io.ReadAll(mv.Reader()): got %v, want no error", err) + } + + if want := []byte(hdrwant); !bytes.Equal(got, want) { + t.Fatalf("mv.Read(): got %q, want %q", got, want) + } +} + +func TestRequestView(t *testing.T) { + body := strings.NewReader("body content") + req, err := http.NewRequest("GET", "http://example.com/path?k=v", body) + if err != nil { + t.Fatalf("http.NewRequest(): got %v, want no error", err) + } + req.Header.Set("Request-Header", "true") + + // Force Content Length to be unset to simulate lack of Content-Length and + // Transfer-Encoding which is valid. + req.ContentLength = -1 + + mv := New() + if err := mv.SnapshotRequest(req); err != nil { + t.Fatalf("SnapshotRequest(): got %v, want no error", err) + } + + got, err := io.ReadAll(mv.HeaderReader()) + if err != nil { + t.Fatalf("io.ReadAll(mv.HeaderReader()): got %v, want no error", err) + } + + hdrwant := "GET http://example.com/path?k=v HTTP/1.1\r\n" + + "Host: example.com\r\n" + + "Request-Header: true\r\n\r\n" + + if !bytes.Equal(got, []byte(hdrwant)) { + t.Fatalf("mv.HeaderReader(): got %q, want %q", got, hdrwant) + } + + br, err := mv.BodyReader() + if err != nil { + t.Fatalf("mv.BodyReader(): got %v, want no error", err) + } + + got, err = io.ReadAll(br) + if err != nil { + t.Fatalf("io.ReadAll(mv.BodyReader()): got %v, want no error", err) + } + + bodywant := "body content" + if !bytes.Equal(got, []byte(bodywant)) { + t.Fatalf("mv.BodyReader(): got %q, want %q", got, bodywant) + } + + r, err := mv.Reader() + if err != nil { + t.Fatalf("mv.Reader(): got %v, want no error", err) + } + got, err = io.ReadAll(r) + if err != nil { + t.Fatalf("io.ReadAll(mv.Reader()): got %v, want no error", err) + } + + if want := []byte(hdrwant + bodywant); !bytes.Equal(got, want) { + t.Fatalf("mv.Read(): got %q, want %q", got, want) + } + + // Sanity check to ensure it still parses. + if _, err := http.ReadRequest(bufio.NewReader(bytes.NewReader(got))); err != nil { + t.Fatalf("http.ReadRequest(): got %v, want no error", err) + } +} + +func TestRequestViewSkipBodyUnlessContentType(t *testing.T) { + req, err := http.NewRequest("GET", "http://example.com", strings.NewReader("body content")) + if err != nil { + t.Fatalf("http.NewRequest(): got %v, want no error", err) + } + req.ContentLength = 12 + req.Header.Set("Content-Type", "text/plain; charset=utf-8") + + mv := New() + mv.SkipBodyUnlessContentType("text/plain") + if err := mv.SnapshotRequest(req); err != nil { + t.Fatalf("SnapshotRequest(): got %v, want no error", err) + } + + br, err := mv.BodyReader() + if err != nil { + t.Fatalf("mv.BodyReader(): got %v, want no error", err) + } + + got, err := io.ReadAll(br) + if err != nil { + t.Fatalf("io.ReadAll(mv.BodyReader()): got %v, want no error", err) + } + + bodywant := "body content" + if !bytes.Equal(got, []byte(bodywant)) { + t.Fatalf("mv.BodyReader(): got %q, want %q", got, bodywant) + } + + req.Header.Set("Content-Type", "image/png") + mv = New() + mv.SkipBodyUnlessContentType("text/plain") + if err := mv.SnapshotRequest(req); err != nil { + t.Fatalf("SnapshotRequest(): got %v, want no error", err) + } + + br, err = mv.BodyReader() + if err != nil { + t.Fatalf("mv.BodyReader(): got %v, want no error", err) + } + + if _, err := br.Read(nil); err != io.EOF { + t.Fatalf("br.Read(): got %v, want io.EOF", err) + } +} + +func TestRequestViewChunkedTransferEncoding(t *testing.T) { + req, err := http.NewRequest("GET", "http://example.com/path?k=v", strings.NewReader("body content")) + if err != nil { + t.Fatalf("http.NewRequest(): got %v, want no error", err) + } + req.TransferEncoding = []string{"chunked"} + req.Header.Set("Trailer", "Trailer-Header") + req.Trailer = http.Header{ + "Trailer-Header": []string{"true"}, + } + + mv := New() + if err := mv.SnapshotRequest(req); err != nil { + t.Fatalf("SnapshotRequest(): got %v, want no error", err) + } + + got, err := io.ReadAll(mv.HeaderReader()) + if err != nil { + t.Fatalf("io.ReadAll(mv.HeaderReader()): got %v, want no error", err) + } + + hdrwant := "GET http://example.com/path?k=v HTTP/1.1\r\n" + + "Host: example.com\r\n" + + "Transfer-Encoding: chunked\r\n" + + "Trailer: Trailer-Header\r\n\r\n" + + if !bytes.Equal(got, []byte(hdrwant)) { + t.Fatalf("mv.HeaderReader(): got %q, want %q", got, hdrwant) + } + + br, err := mv.BodyReader() + if err != nil { + t.Fatalf("mv.BodyReader(): got %v, want no error", err) + } + + got, err = io.ReadAll(br) + if err != nil { + t.Fatalf("io.ReadAll(mv.BodyReader()): got %v, want no error", err) + } + + bodywant := "c\r\nbody content\r\n0\r\n" + if !bytes.Equal(got, []byte(bodywant)) { + t.Fatalf("mv.BodyReader(): got %q, want %q", got, bodywant) + } + + got, err = io.ReadAll(mv.TrailerReader()) + if err != nil { + t.Fatalf("io.ReadAll(mv.TrailerReader()): got %v, want no error", err) + } + + trailerwant := "Trailer-Header: true\r\n" + if !bytes.Equal(got, []byte(trailerwant)) { + t.Fatalf("mv.TrailerReader(): got %q, want %q", got, trailerwant) + } + + r, err := mv.Reader() + if err != nil { + t.Fatalf("mv.Reader(): got %v, want no error", err) + } + got, err = io.ReadAll(r) + if err != nil { + t.Fatalf("io.ReadAll(mv.Reader()): got %v, want no error", err) + } + + if want := []byte(hdrwant + bodywant + trailerwant); !bytes.Equal(got, want) { + t.Fatalf("mv.Read(): got %q, want %q", got, want) + } + + // Sanity check to ensure it still parses. + if _, err := http.ReadRequest(bufio.NewReader(bytes.NewReader(got))); err != nil { + t.Fatalf("http.ReadRequest(): got %v, want no error", err) + } +} + +func TestRequestViewDecodeGzipContentEncoding(t *testing.T) { + body := new(bytes.Buffer) + gw := gzip.NewWriter(body) + gw.Write([]byte("body content")) + gw.Flush() + gw.Close() + + req, err := http.NewRequest("GET", "http://example.com/path?k=v", body) + if err != nil { + t.Fatalf("http.NewRequest(): got %v, want no error", err) + } + req.TransferEncoding = []string{"chunked"} + req.Header.Set("Content-Encoding", "gzip") + + mv := New() + if err := mv.SnapshotRequest(req); err != nil { + t.Fatalf("SnapshotRequest(): got %v, want no error", err) + } + + got, err := io.ReadAll(mv.HeaderReader()) + if err != nil { + t.Fatalf("io.ReadAll(mv.HeaderReader()): got %v, want no error", err) + } + + hdrwant := "GET http://example.com/path?k=v HTTP/1.1\r\n" + + "Host: example.com\r\n" + + "Transfer-Encoding: chunked\r\n" + + "Content-Encoding: gzip\r\n\r\n" + + if !bytes.Equal(got, []byte(hdrwant)) { + t.Fatalf("mv.HeaderReader(): got %q, want %q", got, hdrwant) + } + + br, err := mv.BodyReader(Decode()) + if err != nil { + t.Fatalf("mv.BodyReader(): got %v, want no error", err) + } + + got, err = io.ReadAll(br) + if err != nil { + t.Fatalf("io.ReadAll(mv.BodyReader()): got %v, wt o error", err) + } + + bodywant := "body content" + + if !bytes.Equal(got, []byte(bodywant)) { + t.Fatalf("mv.BodyReader(): got %q, want %q", got, bodywant) + } + + r, err := mv.Reader(Decode()) + if err != nil { + t.Fatalf("mv.Reader(): got %v, want no error", err) + } + got, err = io.ReadAll(r) + if err != nil { + t.Fatalf("io.ReadAll(mv.Reader()): got %v, want no error", err) + } + + if want := []byte(hdrwant + bodywant + "\r\n"); !bytes.Equal(got, want) { + t.Fatalf("mv.Read(): got %q, want %q", got, want) + } +} + +func TestRequestViewDecodeDeflateContentEncoding(t *testing.T) { + body := new(bytes.Buffer) + dw, err := flate.NewWriter(body, -1) + if err != nil { + t.Fatalf("flate.NewWriter(): got %v, want no error", err) + } + dw.Write([]byte("body content")) + dw.Flush() + dw.Close() + + req, err := http.NewRequest("GET", "http://example.com/path?k=v", body) + if err != nil { + t.Fatalf("http.NewRequest(): got %v, want no error", err) + } + req.TransferEncoding = []string{"chunked"} + req.Header.Set("Content-Encoding", "deflate") + + mv := New() + if err := mv.SnapshotRequest(req); err != nil { + t.Fatalf("SnapshotRequest(): got %v, want no error", err) + } + + got, err := io.ReadAll(mv.HeaderReader()) + if err != nil { + t.Fatalf("io.ReadAll(mv.HeaderReader()): got %v, want no error", err) + } + + hdrwant := "GET http://example.com/path?k=v HTTP/1.1\r\n" + + "Host: example.com\r\n" + + "Transfer-Encoding: chunked\r\n" + + "Content-Encoding: deflate\r\n\r\n" + + if !bytes.Equal(got, []byte(hdrwant)) { + t.Fatalf("mv.HeaderReader(): got %q, want %q", got, hdrwant) + } + + br, err := mv.BodyReader(Decode()) + if err != nil { + t.Fatalf("mv.BodyReader(): got %v, want no error", err) + } + + got, err = io.ReadAll(br) + if err != nil { + t.Fatalf("io.ReadAll(mv.BodyReader()): got %v, want no error", err) + } + + bodywant := "body content" + + if !bytes.Equal(got, []byte(bodywant)) { + t.Fatalf("mv.BodyReader(): got %q, want %q", got, bodywant) + } + + r, err := mv.Reader(Decode()) + if err != nil { + t.Fatalf("mv.Reader(): got %v, want no error", err) + } + got, err = io.ReadAll(r) + if err != nil { + t.Fatalf("io.ReadAll(mv.Reader()): got %v, want no error", err) + } + + if want := []byte(hdrwant + bodywant + "\r\n"); !bytes.Equal(got, want) { + t.Fatalf("mv.Read(): got %q, want %q", got, want) + } +} + +func TestResponseViewHeadersOnly(t *testing.T) { + body := strings.NewReader("body content") + res := proxy.NewResponse(200, body, nil) + res.ContentLength = 12 + res.Header.Set("Response-Header", "true") + + mv := New() + mv.SkipBody(true) + if err := mv.SnapshotResponse(res); err != nil { + t.Fatalf("SnapshotResponse(): got %v, want no error", err) + } + + got, err := io.ReadAll(mv.HeaderReader()) + if err != nil { + t.Fatalf("io.ReadAll(mv.HeaderReader()): got %v, want no error", err) + } + + hdrwant := "HTTP/1.1 200 OK\r\n" + + "Content-Length: 12\r\n" + + "Response-Header: true\r\n\r\n" + + if !bytes.Equal(got, []byte(hdrwant)) { + t.Fatalf("mv.HeaderReader(): got %q, want %q", got, hdrwant) + } + + br, err := mv.BodyReader() + if err != nil { + t.Fatalf("mv.BodyReader(): got %v, want no error", err) + } + + if _, err := br.Read(nil); err != io.EOF { + t.Fatalf("io.ReadAll(mv.BodyReader()): got %v, want io.EOF", err) + } + + r, err := mv.Reader() + if err != nil { + t.Fatalf("mv.Reader(): got %v, want no error", err) + } + got, err = io.ReadAll(r) + if err != nil { + t.Fatalf("io.ReadAll(mv.Reader()): got %v, want no error", err) + } + + if want := []byte(hdrwant); !bytes.Equal(got, want) { + t.Fatalf("mv.Read(): got %q, want %q", got, want) + } +} + +func TestResponseView(t *testing.T) { + body := strings.NewReader("body content") + res := proxy.NewResponse(200, body, nil) + res.ContentLength = 12 + res.Header.Set("Response-Header", "true") + + mv := New() + if err := mv.SnapshotResponse(res); err != nil { + t.Fatalf("SnapshotResponse(): got %v, want no error", err) + } + + got, err := io.ReadAll(mv.HeaderReader()) + if err != nil { + t.Fatalf("io.ReadAll(mv.HeaderReader()): got %v, want no error", err) + } + + hdrwant := "HTTP/1.1 200 OK\r\n" + + "Content-Length: 12\r\n" + + "Response-Header: true\r\n\r\n" + + if !bytes.Equal(got, []byte(hdrwant)) { + t.Fatalf("mv.HeaderReader(): got %q, want %q", got, hdrwant) + } + + br, err := mv.BodyReader() + if err != nil { + t.Fatalf("mv.BodyReader(): got %v, want no error", err) + } + + got, err = io.ReadAll(br) + if err != nil { + t.Fatalf("io.ReadAll(mv.BodyReader()): got %v, want no error", err) + } + + bodywant := "body content" + if !bytes.Equal(got, []byte(bodywant)) { + t.Fatalf("mv.BodyReader(): got %q, want %q", got, bodywant) + } + + r, err := mv.Reader() + if err != nil { + t.Fatalf("mv.Reader(): got %v, want no error", err) + } + got, err = io.ReadAll(r) + if err != nil { + t.Fatalf("io.ReadAll(mv.Reader()): got %v, want no error", err) + } + + if want := []byte(hdrwant + bodywant); !bytes.Equal(got, want) { + t.Fatalf("mv.Read(): got %q, want %q", got, want) + } + + // Sanity check to ensure it still parses. + if _, err := http.ReadResponse(bufio.NewReader(bytes.NewReader(got)), nil); err != nil { + t.Fatalf("http.ReadResponse(): got %v, want no error", err) + } +} + +func TestResponseViewSkipBodyUnlessContentType(t *testing.T) { + res := proxy.NewResponse(200, strings.NewReader("body content"), nil) + res.ContentLength = 12 + res.Header.Set("Content-Type", "text/plain; charset=utf-8") + + mv := New() + mv.SkipBodyUnlessContentType("text/plain") + if err := mv.SnapshotResponse(res); err != nil { + t.Fatalf("SnapshotResponse(): got %v, want no error", err) + } + + br, err := mv.BodyReader() + if err != nil { + t.Fatalf("mv.BodyReader(): got %v, want no error", err) + } + + got, err := io.ReadAll(br) + if err != nil { + t.Fatalf("io.ReadAll(mv.BodyReader()): got %v, want no error", err) + } + + bodywant := "body content" + if !bytes.Equal(got, []byte(bodywant)) { + t.Fatalf("mv.BodyReader(): got %q, want %q", got, bodywant) + } + + res.Header.Set("Content-Type", "image/png") + mv = New() + mv.SkipBodyUnlessContentType("text/plain") + if err := mv.SnapshotResponse(res); err != nil { + t.Fatalf("SnapshotResponse(): got %v, want no error", err) + } + + br, err = mv.BodyReader() + if err != nil { + t.Fatalf("mv.BodyReader(): got %v, want no error", err) + } + + if _, err := br.Read(nil); err != io.EOF { + t.Fatalf("br.Read(): got %v, want io.EOF", err) + } +} + +func TestResponseViewChunkedTransferEncoding(t *testing.T) { + body := strings.NewReader("body content") + res := proxy.NewResponse(200, body, nil) + res.TransferEncoding = []string{"chunked"} + res.Header.Set("Trailer", "Trailer-Header") + res.Trailer = http.Header{ + "Trailer-Header": []string{"true"}, + } + + mv := New() + if err := mv.SnapshotResponse(res); err != nil { + t.Fatalf("SnapshotResponse(): got %v, want no error", err) + } + + got, err := io.ReadAll(mv.HeaderReader()) + if err != nil { + t.Fatalf("io.ReadAll(mv.HeaderReader()): got %v, want no error", err) + } + + hdrwant := "HTTP/1.1 200 OK\r\n" + + "Transfer-Encoding: chunked\r\n" + + "Trailer: Trailer-Header\r\n\r\n" + + if !bytes.Equal(got, []byte(hdrwant)) { + t.Fatalf("mv.HeaderReader(): got %q, want %q", got, hdrwant) + } + + br, err := mv.BodyReader() + if err != nil { + t.Fatalf("mv.BodyReader(): got %v, want no error", err) + } + + got, err = io.ReadAll(br) + if err != nil { + t.Fatalf("io.ReadAll(mv.BodyReader()): got %v, want no error", err) + } + + bodywant := "c\r\nbody content\r\n0\r\n" + if !bytes.Equal(got, []byte(bodywant)) { + t.Fatalf("mv.BodyReader(): got %q, want %q", got, bodywant) + } + + got, err = io.ReadAll(mv.TrailerReader()) + if err != nil { + t.Fatalf("io.ReadAll(mv.TrailerReader()): got %v, want no error", err) + } + + trailerwant := "Trailer-Header: true\r\n" + if !bytes.Equal(got, []byte(trailerwant)) { + t.Fatalf("mv.TrailerReader(): got %q, want %q", got, trailerwant) + } + + r, err := mv.Reader() + if err != nil { + t.Fatalf("mv.Reader(): got %v, want no error", err) + } + got, err = io.ReadAll(r) + if err != nil { + t.Fatalf("io.ReadAll(mv.Reader()): got %v, want no error", err) + } + + if want := []byte(hdrwant + bodywant + trailerwant); !bytes.Equal(got, want) { + t.Fatalf("mv.Read(): got %q, want %q", got, want) + } + + // Sanity check to ensure it still parses. + if _, err := http.ReadResponse(bufio.NewReader(bytes.NewReader(got)), nil); err != nil { + t.Fatalf("http.ReadResponse(): got %v, want no error", err) + } +} + +func TestResponseViewDecodeGzipContentEncoding(t *testing.T) { + body := new(bytes.Buffer) + gw := gzip.NewWriter(body) + gw.Write([]byte("body content")) + gw.Flush() + gw.Close() + + res := proxy.NewResponse(200, body, nil) + res.TransferEncoding = []string{"chunked"} + res.Header.Set("Content-Encoding", "gzip") + + mv := New() + if err := mv.SnapshotResponse(res); err != nil { + t.Fatalf("SnapshotResponse(): got %v, want no error", err) + } + + got, err := io.ReadAll(mv.HeaderReader()) + if err != nil { + t.Fatalf("io.ReadAll(mv.HeaderReader()): got %v, want no error", err) + } + + hdrwant := "HTTP/1.1 200 OK\r\n" + + "Transfer-Encoding: chunked\r\n" + + "Content-Encoding: gzip\r\n\r\n" + + if !bytes.Equal(got, []byte(hdrwant)) { + t.Fatalf("mv.HeaderReader(): got %q, want %q", got, hdrwant) + } + + br, err := mv.BodyReader(Decode()) + if err != nil { + t.Fatalf("mv.BodyReader(): got %v, want no error", err) + } + + got, err = io.ReadAll(br) + if err != nil { + t.Fatalf("io.ReadAll(mv.BodyReader()): got %v, wt o error", err) + } + + bodywant := "body content" + + if !bytes.Equal(got, []byte(bodywant)) { + t.Fatalf("mv.BodyReader(): got %q, want %q", got, bodywant) + } + + r, err := mv.Reader(Decode()) + if err != nil { + t.Fatalf("mv.Reader(): got %v, want no error", err) + } + got, err = io.ReadAll(r) + if err != nil { + t.Fatalf("io.ReadAll(mv.Reader()): got %v, want no error", err) + } + + if want := []byte(hdrwant + bodywant + "\r\n"); !bytes.Equal(got, want) { + t.Fatalf("mv.Read(): got %q, want %q", got, want) + } +} + +func TestResponseViewDecodeGzipContentEncodingPartial(t *testing.T) { + bodywant := "partial content" + res := proxy.NewResponse(206, strings.NewReader(bodywant), nil) + res.TransferEncoding = []string{"chunked"} + res.Header.Set("Content-Encoding", "gzip") + + mv := New() + if err := mv.SnapshotResponse(res); err != nil { + t.Fatalf("SnapshotResponse(): got %v, want no error", err) + } + br, err := mv.BodyReader(Decode()) + if err != nil { + t.Fatalf("mv.BodyReader(): got %v, want no error", err) + } + + got, err := io.ReadAll(br) + if err != nil { + t.Fatalf("io.ReadAll(mv.BodyReader()): got %v, wt o error", err) + } + if !bytes.Equal(got, []byte(bodywant)) { + t.Fatalf("mv.BodyReader(): got %q, want %q", got, bodywant) + } +} + +func TestResponseViewDecodeDeflateContentEncoding(t *testing.T) { + body := new(bytes.Buffer) + dw, err := flate.NewWriter(body, -1) + if err != nil { + t.Fatalf("flate.NewWriter(): got %v, want no error", err) + } + dw.Write([]byte("body content")) + dw.Flush() + dw.Close() + + res := proxy.NewResponse(200, body, nil) + res.TransferEncoding = []string{"chunked"} + res.Header.Set("Content-Encoding", "deflate") + + mv := New() + if err := mv.SnapshotResponse(res); err != nil { + t.Fatalf("SnapshotResponse(): got %v, want no error", err) + } + + got, err := io.ReadAll(mv.HeaderReader()) + if err != nil { + t.Fatalf("io.ReadAll(mv.HeaderReader()): got %v, want no error", err) + } + + hdrwant := "HTTP/1.1 200 OK\r\n" + + "Transfer-Encoding: chunked\r\n" + + "Content-Encoding: deflate\r\n\r\n" + + if !bytes.Equal(got, []byte(hdrwant)) { + t.Fatalf("mv.HeaderReader(): got %q, want %q", got, hdrwant) + } + + br, err := mv.BodyReader(Decode()) + if err != nil { + t.Fatalf("mv.BodyReader(): got %v, want no error", err) + } + + got, err = io.ReadAll(br) + if err != nil { + t.Fatalf("io.ReadAll(mv.BodyReader()): got %v, wt o error", err) + } + + bodywant := "body content" + + if !bytes.Equal(got, []byte(bodywant)) { + t.Fatalf("mv.BodyReader(): got %q, want %q", got, bodywant) + } + + r, err := mv.Reader(Decode()) + if err != nil { + t.Fatalf("mv.Reader(): got %v, want no error", err) + } + got, err = io.ReadAll(r) + if err != nil { + t.Fatalf("io.ReadAll(mv.Reader()): got %v, want no error", err) + } + + if want := []byte(hdrwant + bodywant + "\r\n"); !bytes.Equal(got, want) { + t.Fatalf("mv.Read(): got %q, want %q", got, want) + } +} From 1e8a5856eaf8b180f1acac949053c929b1e31ba0 Mon Sep 17 00:00:00 2001 From: CFC4N Date: Tue, 17 Sep 2024 23:04:06 +0800 Subject: [PATCH 4/8] pkg: add package `util/proxy` Signed-off-by: CFC4N --- pkg/util/proxy/header.go | 196 ++++++++++++++++++ pkg/util/proxy/header_test.go | 344 +++++++++++++++++++++++++++++++ pkg/util/proxy/proxyutil.go | 103 +++++++++ pkg/util/proxy/proxyutil_test.go | 88 ++++++++ 4 files changed, 731 insertions(+) create mode 100644 pkg/util/proxy/header.go create mode 100644 pkg/util/proxy/header_test.go create mode 100644 pkg/util/proxy/proxyutil.go create mode 100644 pkg/util/proxy/proxyutil_test.go diff --git a/pkg/util/proxy/header.go b/pkg/util/proxy/header.go new file mode 100644 index 000000000..79f77a654 --- /dev/null +++ b/pkg/util/proxy/header.go @@ -0,0 +1,196 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package proxy + +import ( + "fmt" + "net/http" + "strconv" +) + +// Header is a generic representation of a set of HTTP headers for requests and +// responses. +type Header struct { + h http.Header + + host func() string + cl func() int64 + te func() []string + + setHost func(string) + setCL func(int64) + setTE func([]string) +} + +// RequestHeader returns a new set of headers from a request. +func RequestHeader(req *http.Request) *Header { + return &Header{ + h: req.Header, + host: func() string { return req.Host }, + cl: func() int64 { return req.ContentLength }, + te: func() []string { return req.TransferEncoding }, + setHost: func(host string) { req.Host = host }, + setCL: func(cl int64) { req.ContentLength = cl }, + setTE: func(te []string) { req.TransferEncoding = te }, + } +} + +// ResponseHeader returns a new set of headers from a request. +func ResponseHeader(res *http.Response) *Header { + return &Header{ + h: res.Header, + host: func() string { return "" }, + cl: func() int64 { return res.ContentLength }, + te: func() []string { return res.TransferEncoding }, + setHost: func(string) {}, + setCL: func(cl int64) { res.ContentLength = cl }, + setTE: func(te []string) { res.TransferEncoding = te }, + } +} + +// Set sets value at header name for the request or response. +func (h *Header) Set(name, value string) error { + switch http.CanonicalHeaderKey(name) { + case "Host": + h.setHost(value) + case "Content-Length": + cl, err := strconv.ParseInt(value, 10, 64) + if err != nil { + return err + } + + h.setCL(cl) + case "Transfer-Encoding": + h.setTE([]string{value}) + default: + h.h.Set(name, value) + } + + return nil +} + +// Add appends the value to the existing header at name for the request or +// response. +func (h *Header) Add(name, value string) error { + switch http.CanonicalHeaderKey(name) { + case "Host": + if h.host() != "" { + return fmt.Errorf("proxyutil: illegal header multiple: %s", "Host") + } + + return h.Set(name, value) + case "Content-Length": + if h.cl() > 0 { + return fmt.Errorf("proxyutil: illegal header multiple: %s", "Content-Length") + } + + return h.Set(name, value) + case "Transfer-Encoding": + h.setTE(append(h.te(), value)) + default: + h.h.Add(name, value) + } + + return nil +} + +// Get returns the first value at header name for the request or response. +func (h *Header) Get(name string) string { + switch http.CanonicalHeaderKey(name) { + case "Host": + return h.host() + case "Content-Length": + if h.cl() < 0 { + return "" + } + + return strconv.FormatInt(h.cl(), 10) + case "Transfer-Encoding": + if len(h.te()) < 1 { + return "" + } + + return h.te()[0] + default: + return h.h.Get(name) + } +} + +// All returns all the values for header name. If the header does not exist it +// returns nil, false. +func (h *Header) All(name string) ([]string, bool) { + switch http.CanonicalHeaderKey(name) { + case "Host": + if h.host() == "" { + return nil, false + } + + return []string{h.host()}, true + case "Content-Length": + if h.cl() <= 0 { + return nil, false + } + + return []string{strconv.FormatInt(h.cl(), 10)}, true + case "Transfer-Encoding": + if h.te() == nil { + return nil, false + } + + return h.te(), true + default: + vs, ok := h.h[http.CanonicalHeaderKey(name)] + return vs, ok + } +} + +// Del deletes the header at name for the request or response. +func (h *Header) Del(name string) { + switch http.CanonicalHeaderKey(name) { + case "Host": + h.setHost("") + case "Content-Length": + h.setCL(-1) + case "Transfer-Encoding": + h.setTE(nil) + default: + h.h.Del(name) + } +} + +// Map returns an http.Header that includes Host, Content-Length, and +// Transfer-Encoding. +func (h *Header) Map() http.Header { + hm := make(http.Header) + + for k, vs := range h.h { + hm[k] = vs + } + + for _, k := range []string{ + "Host", + "Content-Length", + "Transfer-Encoding", + } { + vs, ok := h.All(k) + if !ok { + continue + } + + hm[k] = vs + } + + return hm +} diff --git a/pkg/util/proxy/header_test.go b/pkg/util/proxy/header_test.go new file mode 100644 index 000000000..0be1d4b99 --- /dev/null +++ b/pkg/util/proxy/header_test.go @@ -0,0 +1,344 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package proxy + +import ( + "net/http" + "reflect" + "testing" +) + +func TestRequestHeader(t *testing.T) { + req, err := http.NewRequest("GET", "http://example.com", nil) + if err != nil { + t.Fatalf("http.NewRequest(): got %v, want no error", err) + } + + h := RequestHeader(req) + + tt := []struct { + name string + value string + }{ + { + name: "Host", + value: "example.com", + }, + { + name: "Test-Header", + value: "true", + }, + { + name: "Content-Length", + value: "100", + }, + { + name: "Transfer-Encoding", + value: "chunked", + }, + } + + for i, tc := range tt { + if err := h.Set(tc.name, tc.value); err != nil { + t.Errorf("%d. h.Set(%q, %q): got %v, want no error", i, tc.name, tc.value, err) + } + } + + if got, want := req.Host, "example.com"; got != want { + t.Errorf("req.Host: got %q, want %q", got, want) + } + if got, want := req.Header.Get("Test-Header"), "true"; got != want { + t.Errorf("req.Header.Get(%q): got %q, want %q", "Test-Header", got, want) + } + if got, want := req.ContentLength, int64(100); got != want { + t.Errorf("req.ContentLength: got %d, want %d", got, want) + } + if got, want := req.TransferEncoding, []string{"chunked"}; !reflect.DeepEqual(got, want) { + t.Errorf("req.TransferEncoding: got %v, want %v", got, want) + } + + if got, want := len(h.Map()), 4; got != want { + t.Errorf("h.Map(): got %d entries, want %d entries", got, want) + } + + for n, vs := range h.Map() { + var want string + switch n { + case "Host": + want = "example.com" + case "Content-Length": + want = "100" + case "Transfer-Encoding": + want = "chunked" + case "Test-Header": + want = "true" + default: + t.Errorf("h.Map(): got unexpected %s header", n) + } + + if got := vs[0]; got != want { + t.Errorf("h.Map(): got %s header with value %s, want value %s", n, got, want) + } + } + + for i, tc := range tt { + got, ok := h.All(tc.name) + if !ok { + t.Errorf("%d. h.All(%q): got false, want true", i, tc.name) + } + + if want := []string{tc.value}; !reflect.DeepEqual(got, want) { + t.Errorf("%d. h.All(%q): got %v, want %v", i, tc.name, got, want) + } + + if got, want := h.Get(tc.name), tc.value; got != want { + t.Errorf("%d. h.Get(%q): got %q, want %q", i, tc.name, got, want) + } + + h.Del(tc.name) + } + + if got, want := req.Host, ""; got != want { + t.Errorf("req.Host: got %q, want %q", got, want) + } + if got, want := req.Header.Get("Test-Header"), ""; got != want { + t.Errorf("req.Header.Get(%q): got %q, want %q", "Test-Header", got, want) + } + if got, want := req.ContentLength, int64(-1); got != want { + t.Errorf("req.ContentLength: got %d, want %d", got, want) + } + if got := req.TransferEncoding; got != nil { + t.Errorf("req.TransferEncoding: got %v, want nil", got) + } + + for i, tc := range tt { + if got, want := h.Get(tc.name), ""; got != want { + t.Errorf("%d. h.Get(%q): got %q, want %q", i, tc.name, got, want) + } + + got, ok := h.All(tc.name) + if ok { + t.Errorf("%d. h.All(%q): got ok, want !ok", i, tc.name) + } + if got != nil { + t.Errorf("%d. h.All(%q): got %v, want nil", i, tc.name, got) + } + } +} + +func TestRequestHeaderAdd(t *testing.T) { + req, err := http.NewRequest("GET", "http://example.com", nil) + if err != nil { + t.Fatalf("http.NewRequest(): got %v, want no error", err) + } + req.Host = "" // Set to empty so add may overwrite. + + h := RequestHeader(req) + + tt := []struct { + name string + values []string + errOnSecondValue bool + }{ + { + name: "Host", + values: []string{"example.com", "invalid.com"}, + errOnSecondValue: true, + }, + { + name: "Test-Header", + values: []string{"first", "second"}, + }, + { + name: "Content-Length", + values: []string{"100", "101"}, + errOnSecondValue: true, + }, + { + name: "Transfer-Encoding", + values: []string{"chunked", "gzip"}, + }, + } + + for i, tc := range tt { + if err := h.Add(tc.name, tc.values[0]); err != nil { + t.Errorf("%d. h.Add(%q, %q): got %v, want no error", i, tc.name, tc.values[0], err) + } + if err := h.Add(tc.name, tc.values[1]); err != nil && !tc.errOnSecondValue { + t.Errorf("%d. h.Add(%q, %q): got %v, want no error", i, tc.name, tc.values[1], err) + } + } + + if got, want := req.Host, "example.com"; got != want { + t.Errorf("req.Host: got %q, want %q", got, want) + } + if got, want := req.Header["Test-Header"], []string{"first", "second"}; !reflect.DeepEqual(got, want) { + t.Errorf("req.Header[%q]: got %v, want %v", "Test-Header", got, want) + } + if got, want := req.ContentLength, int64(100); got != want { + t.Errorf("req.ContentLength: got %d, want %d", got, want) + } + if got, want := req.TransferEncoding, []string{"chunked", "gzip"}; !reflect.DeepEqual(got, want) { + t.Errorf("req.TransferEncoding: got %v, want %v", got, want) + } +} + +func TestResponseHeader(t *testing.T) { + res := NewResponse(200, nil, nil) + + h := ResponseHeader(res) + + tt := []struct { + name string + value string + }{ + { + name: "Test-Header", + value: "true", + }, + { + name: "Content-Length", + value: "100", + }, + { + name: "Transfer-Encoding", + value: "chunked", + }, + } + + for i, tc := range tt { + if err := h.Set(tc.name, tc.value); err != nil { + t.Errorf("%d. h.Set(%q, %q): got %v, want no error", i, tc.name, tc.value, err) + } + } + + if got, want := res.Header.Get("Test-Header"), "true"; got != want { + t.Errorf("res.Header.Get(%q): got %q, want %q", "Test-Header", got, want) + } + if got, want := res.ContentLength, int64(100); got != want { + t.Errorf("res.ContentLength: got %d, want %d", got, want) + } + if got, want := res.TransferEncoding, []string{"chunked"}; !reflect.DeepEqual(got, want) { + t.Errorf("res.TransferEncoding: got %v, want %v", got, want) + } + + if got, want := len(h.Map()), 3; got != want { + t.Errorf("h.Map(): got %d entries, want %d entries", got, want) + } + + for n, vs := range h.Map() { + var want string + switch n { + case "Content-Length": + want = "100" + case "Transfer-Encoding": + want = "chunked" + case "Test-Header": + want = "true" + default: + t.Errorf("h.Map(): got unexpected %s header", n) + } + + if got := vs[0]; got != want { + t.Errorf("h.Map(): got %s header with value %s, want value %s", n, got, want) + } + } + + for i, tc := range tt { + got, ok := h.All(tc.name) + if !ok { + t.Errorf("%d. h.All(%q): got false, want true", i, tc.name) + } + + if want := []string{tc.value}; !reflect.DeepEqual(got, want) { + t.Errorf("%d. h.All(%q): got %v, want %v", i, tc.name, got, want) + } + + if got, want := h.Get(tc.name), tc.value; got != want { + t.Errorf("%d. h.Get(%q): got %q, want %q", i, tc.name, got, want) + } + + h.Del(tc.name) + } + + if got, want := res.Header.Get("Test-Header"), ""; got != want { + t.Errorf("res.Header.Get(%q): got %q, want %q", "Test-Header", got, want) + } + if got, want := res.ContentLength, int64(-1); got != want { + t.Errorf("res.ContentLength: got %d, want %d", got, want) + } + if got := res.TransferEncoding; got != nil { + t.Errorf("res.TransferEncoding: got %v, want nil", got) + } + + for i, tc := range tt { + if got, want := h.Get(tc.name), ""; got != want { + t.Errorf("%d. h.Get(%q): got %q, want %q", i, tc.name, got, want) + } + + got, ok := h.All(tc.name) + if ok { + t.Errorf("%d. h.All(%q): got ok, want !ok", i, tc.name) + } + if got != nil { + t.Errorf("%d. h.All(%q): got %v, want nil", i, tc.name, got) + } + } +} + +func TestResponseHeaderAdd(t *testing.T) { + res := NewResponse(200, nil, nil) + + h := ResponseHeader(res) + + tt := []struct { + name string + values []string + errOnSecondValue bool + }{ + { + name: "Test-Header", + values: []string{"first", "second"}, + }, + { + name: "Content-Length", + values: []string{"100", "101"}, + errOnSecondValue: true, + }, + { + name: "Transfer-Encoding", + values: []string{"chunked", "gzip"}, + }, + } + + for i, tc := range tt { + if err := h.Add(tc.name, tc.values[0]); err != nil { + t.Errorf("%d. h.Add(%q, %q): got %v, want no error", i, tc.name, tc.values[0], err) + } + if err := h.Add(tc.name, tc.values[1]); err != nil && !tc.errOnSecondValue { + t.Errorf("%d. h.Add(%q, %q): got %v, want no error", i, tc.name, tc.values[1], err) + } + } + + if got, want := res.Header["Test-Header"], []string{"first", "second"}; !reflect.DeepEqual(got, want) { + t.Errorf("res.Header[%q]: got %v, want %v", "Test-Header", got, want) + } + if got, want := res.ContentLength, int64(100); got != want { + t.Errorf("res.ContentLength: got %d, want %d", got, want) + } + if got, want := res.TransferEncoding, []string{"chunked", "gzip"}; !reflect.DeepEqual(got, want) { + t.Errorf("res.TransferEncoding: got %v, want %v", got, want) + } +} diff --git a/pkg/util/proxy/proxyutil.go b/pkg/util/proxy/proxyutil.go new file mode 100644 index 000000000..7c27facea --- /dev/null +++ b/pkg/util/proxy/proxyutil.go @@ -0,0 +1,103 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/* +Package proxyutil provides functionality for building proxies. +*/ +package proxy + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "net/http" + "regexp" + "strconv" + "strings" + "time" +) + +// NewResponse builds new HTTP responses. +// If body is nil, an empty byte.Buffer will be provided to be consistent with +// the guarantees provided by http.Transport and http.Client. +func NewResponse(code int, body io.Reader, req *http.Request) *http.Response { + if body == nil { + body = &bytes.Buffer{} + } + + rc, ok := body.(io.ReadCloser) + if !ok { + rc = ioutil.NopCloser(body) + } + + res := &http.Response{ + StatusCode: code, + Status: fmt.Sprintf("%d %s", code, http.StatusText(code)), + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + Header: http.Header{}, + Body: rc, + Request: req, + } + + if req != nil { + res.Close = req.Close + res.Proto = req.Proto + res.ProtoMajor = req.ProtoMajor + res.ProtoMinor = req.ProtoMinor + } + + return res +} + +// Warning adds an error to the Warning header in the format: 199 "martian" +// "error message" "date". +func Warning(header http.Header, err error) { + date := header.Get("Date") + if date == "" { + date = time.Now().Format(http.TimeFormat) + } + + w := fmt.Sprintf(`199 "martian" %q %q`, err.Error(), date) + header.Add("Warning", w) +} + +// GetRangeStart returns the byte index of the start of the range, if it has one. +// Returns 0 if the range header is absent, and -1 if the range header is invalid or +// has multi-part ranges. +func GetRangeStart(res *http.Response) int64 { + if res.StatusCode != http.StatusPartialContent { + return 0 + } + + if strings.Contains(res.Header.Get("Content-Type"), "multipart/byteranges") { + return -1 + } + + re := regexp.MustCompile(`bytes (\d+)-\d+/\d+`) + matchSlice := re.FindStringSubmatch(res.Header.Get("Content-Range")) + + if len(matchSlice) < 2 { + return -1 + } + + num, err := strconv.ParseInt(matchSlice[1], 10, 64) + + if err != nil { + return -1 + } + return num +} diff --git a/pkg/util/proxy/proxyutil_test.go b/pkg/util/proxy/proxyutil_test.go new file mode 100644 index 000000000..8464195e2 --- /dev/null +++ b/pkg/util/proxy/proxyutil_test.go @@ -0,0 +1,88 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package proxy + +import ( + "fmt" + "io" + "net/http" + "strings" + "testing" +) + +func TestNewResponse(t *testing.T) { + req, err := http.NewRequest("GET", "http://www.example.com", nil) + if err != nil { + t.Fatalf("http.NewRequest(): got %v, want no error", err) + } + req.Close = true + + res := NewResponse(200, nil, req) + if got, want := res.StatusCode, 200; got != want { + t.Errorf("res.StatusCode: got %d, want %d", got, want) + } + if got, want := res.Status, "200 OK"; got != want { + t.Errorf("res.Status: got %q, want %q", got, want) + } + if !res.Close { + t.Error("res.Close: got false, want true") + } + if got, want := res.Proto, "HTTP/1.1"; got != want { + t.Errorf("res.Proto: got %q, want %q", got, want) + } + if got, want := res.ProtoMajor, 1; got != want { + t.Errorf("res.ProtoMajor: got %d, want %d", got, want) + } + if got, want := res.ProtoMinor, 1; got != want { + t.Errorf("res.ProtoMinor: got %d, want %d", got, want) + } + if res.Header == nil { + t.Error("res.Header: got nil, want header") + } + if _, ok := res.Body.(io.ReadCloser); !ok { + t.Error("res.Body.(io.ReadCloser): got !ok, want ok") + } + if got, want := res.Request, req; got != want { + t.Errorf("res.Request: got %v, want %v", got, want) + } +} + +func TestWarning(t *testing.T) { + hdr := http.Header{} + err := fmt.Errorf("modifier error") + + Warning(hdr, err) + + if got, want := len(hdr["Warning"]), 1; got != want { + t.Fatalf("len(hdr[%q]): got %d, want %d", "Warning", got, want) + } + + want := `199 "martian" "modifier error"` + if got := hdr["Warning"][0]; !strings.HasPrefix(got, want) { + t.Errorf("hdr[%q][0]: got %q, want to have prefix %q", "Warning", got, want) + } + + hdr.Set("Date", "Mon, 02 Jan 2006 15:04:05 GMT") + Warning(hdr, err) + + if got, want := len(hdr["Warning"]), 2; got != want { + t.Fatalf("len(hdr[%q]): got %d, want %d", "Warning", got, want) + } + + want = `199 "martian" "modifier error" "Mon, 02 Jan 2006 15:04:05 GMT"` + if got := hdr["Warning"][1]; got != want { + t.Errorf("hdr[%q][1]: got %q, want %q", "Warning", got, want) + } +} From 6b27c0db38c8e39ac437b5eccb21219715fe0ad0 Mon Sep 17 00:00:00 2001 From: CFC4N Date: Sat, 28 Sep 2024 07:39:12 +0800 Subject: [PATCH 5/8] pkg: remove refs to deprecated io/ioutil update readme Signed-off-by: CFC4N --- README.md | 28 +++++++++------------------- README_CN.md | 10 ++++------ pkg/har/har.go | 5 ++--- pkg/util/proxy/proxyutil.go | 3 +-- 4 files changed, 16 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 4410e4481..31dd6dc20 100644 --- a/README.md +++ b/README.md @@ -9,8 +9,7 @@ ### eCapture(旁观者): capture SSL/TLS text content without a CA certificate using eBPF. -> **Note:** -> +> [!IMPORTANT] > Supports Linux/Android kernel versions x86_64 4.18 and above, **aarch64 5.5** and above. > Need ROOT permission. > Does not support Windows and macOS system. @@ -49,7 +48,7 @@ ### ELF binary file -> **Note** +> [!TIP] > support Linux/Android x86_64/aarch64. Download ELF zip file [release](https://github.com/gojue/ecapture/releases) , unzip and use by @@ -57,7 +56,7 @@ command `sudo ecapture --help`. ### Docker image -> **Note** +> [!TIP] > Linux only. ```shell @@ -178,6 +177,12 @@ The OpenSSL module supports three capture modes: Supported TLS encrypted http `1.0/1.1/2.0` over TCP, and http3 `QUIC` protocol over UDP. You can specify `-m pcap` or `-m pcapng` and use it in conjunction with `--pcapfile` and `-i` parameters. The default value for `--pcapfile` is `ecapture_openssl.pcapng`. +```shell +sudo ecapture tls -m pcap -i eth0 --pcapfile=ecapture.pcapng tcp port 443 +``` + +This command saves captured plaintext data packets as a pcapng file, which can be viewed using `Wireshark`. + ```shell sudo ecapture tls -m pcap -w ecap.pcapng -i ens160 2024-09-15T06:54:12Z INF AppName="eCapture(旁观者)" @@ -223,12 +228,6 @@ sudo ecapture tls -m pcap -w ecap.pcapng -i ens160 Used `Wireshark` to open `ecap.pcapng` file to view the plaintext data packets. -```shell -sudo ecapture tls -m pcap -i eth0 --pcapfile=ecapture.pcapng tcp port 443 -``` - -This command saves captured plaintext data packets as a pcapng file, which can be viewed using `Wireshark`. - #### Keylog Mode You can specify `-m keylog` or `-m key` and use it in conjunction with the `--keylogfile` parameter, which defaults to `ecapture_masterkey.log`. @@ -254,15 +253,6 @@ SSLKEYLOG information.) Similar to the OpenSSL module. -#### check your server BTF config： - -```shell -cfc4n@vm-server:~$# uname -r -4.18.0-305.3.1.el8.x86_64 -cfc4n@vm-server:~$# cat /boot/config-`uname -r` | grep CONFIG_DEBUG_INFO_BTF -CONFIG_DEBUG_INFO_BTF=y -``` - #### gotls command capture tls text context. diff --git a/README_CN.md b/README_CN.md index 72d4f67f8..38fb96f79 100644 --- a/README_CN.md +++ b/README_CN.md @@ -6,11 +6,11 @@ [![GitHub forks](https://img.shields.io/github/forks/gojue/ecapture?label=Forks&logo=github)](https://github.com/gojue/ecapture) [![CI](https://github.com/gojue/ecapture/actions/workflows/codeql-analysis.yml/badge.svg)](https://github.com/gojue/ecapture/actions/workflows/code-analysis.yml) [![Github Version](https://img.shields.io/github/v/release/gojue/ecapture?display_name=tag&include_prereleases&sort=semver)](https://github.com/gojue/ecapture/releases) +[![QQ 群](https://img.shields.io/badge/QQ群-%2312B7F5?logo=tencent-qq&logoColor=white&style=flat-square)](https://qm.qq.com/cgi-bin/qm/qr?k=iCu561fq4zdbHVdntQLFV0Xugrnf7Hpv&jump_from=webapi&authKey=YamGv189Cg+KFdQt1Qnsw6GZlpx8BYA+G2WZFezohY4M03V+l0eElZWOhZj/wR/5) ### eCapture(旁观者): 基于eBPF技术实现SSL/TLS加密的明文捕获，无需CA证书。 -> **提醒：** -> +> [!TIP] > 支持Linux系统内核x86_64 4.18及以上版本，aarch64 5.5及以上版本； > 需要ROOT权限； > 不支持Windows、macOS系统； @@ -47,16 +47,14 @@ eCapture的中文名字为**旁观者**，即「**当局者迷，旁观者清** ### ELF可执行文件 -> **提醒** -> +> [!IMPORTANT] > 支持 Linux/Android的x86_64/aarch64 CPU架构。 下载 [release](https://github.com/gojue/ecapture/releases) 的二进制包，可直接使用。 ### Docker容器镜像 -> **提醒** -> +> [!TIP] > 仅支持Linux x86_64/aarch64。 ```shell diff --git a/pkg/har/har.go b/pkg/har/har.go index 83378c0b7..5653a825f 100644 --- a/pkg/har/har.go +++ b/pkg/har/har.go @@ -29,7 +29,6 @@ import ( "github.com/gojue/ecapture/pkg/messageview" "github.com/gojue/ecapture/pkg/util/proxy" "io" - "io/ioutil" "mime" "mime/multipart" "net/http" @@ -610,7 +609,7 @@ func NewResponse(res *http.Response, withBody bool) (*Response, error) { return nil, err } - body, err := ioutil.ReadAll(br) + body, err := io.ReadAll(br) if err != nil { return nil, err } @@ -780,7 +779,7 @@ func postData(req *http.Request, logBody bool) (*PostData, error) { } defer p.Close() - body, err := ioutil.ReadAll(p) + body, err := io.ReadAll(p) if err != nil { return nil, err } diff --git a/pkg/util/proxy/proxyutil.go b/pkg/util/proxy/proxyutil.go index 7c27facea..3b0028620 100644 --- a/pkg/util/proxy/proxyutil.go +++ b/pkg/util/proxy/proxyutil.go @@ -21,7 +21,6 @@ import ( "bytes" "fmt" "io" - "io/ioutil" "net/http" "regexp" "strconv" @@ -39,7 +38,7 @@ func NewResponse(code int, body io.Reader, req *http.Request) *http.Response { rc, ok := body.(io.ReadCloser) if !ok { - rc = ioutil.NopCloser(body) + rc = io.NopCloser(body) } res := &http.Response{ From 85d6ad9eb475b965d71128798d210d467afe7ee4 Mon Sep 17 00:00:00 2001 From: CFC4N Date: Tue, 1 Oct 2024 23:29:50 +0800 Subject: [PATCH 6/8] docs: update README Modify the block level of the reminder content div in the Chinese character document. add eCapture roadmap link. Signed-off-by: CFC4N --- README.md | 8 ++++++-- README_CN.md | 19 ++++++++++--------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 31dd6dc20..403c07cfe 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,9 @@ [![GitHub stars](https://img.shields.io/github/stars/gojue/ecapture.svg?label=Stars&logo=github)](https://github.com/gojue/ecapture) [![GitHub forks](https://img.shields.io/github/forks/gojue/ecapture?label=Forks&logo=github)](https://github.com/gojue/ecapture) -[![CI](https://github.com/gojue/ecapture/actions/workflows/codeql-analysis.yml/badge.svg)](https://github.com/gojue/ecapture/actions/workflows/code-analysis.yml) +[![CI](https://github.com/gojue/ecapture/actions/workflows/codeql-analysis.yml/badge.svg)](https://github.com/gojue/ecapture/blob/master/.github/workflows/codeql-analysis.yml) [![Github Version](https://img.shields.io/github/v/release/gojue/ecapture?display_name=tag&include_prereleases&sort=semver)](https://github.com/gojue/ecapture/releases) +[![Docker](https://img.shields.io/docker/pulls/gojue/ecapture?style=flat&logo=docker)](https://github.com/gojue/ecapture) ### eCapture(旁观者): capture SSL/TLS text content without a CA certificate using eBPF. @@ -26,7 +27,7 @@ - [Modules](#modules) - [OpenSSL Module](#openssl-module) - [GoTLS Module](#gotls-module) - - [Other Modules](#bash-module) + - [Other Modules](#other-modules) - [Videos](#videos) - [Contributing](#contributing) - [Compilation](#compilation) @@ -288,6 +289,9 @@ such as `bash\mysqld\postgres` modules, you can use `ecapture -h` to view the li # Contributing See [CONTRIBUTING](./CONTRIBUTING.md) for details on submitting patches and the contribution workflow. +The [eCapture roadmap](https://github.com/orgs/gojue/projects/1) lists many tasks to be developed, and you are also +welcome to participate in the construction. + # Compilation See [COMPILATION](./COMPILATION.md) for details on compiling the eCapture source code. \ No newline at end of file diff --git a/README_CN.md b/README_CN.md index 38fb96f79..eb65f0ac4 100644 --- a/README_CN.md +++ b/README_CN.md @@ -4,16 +4,15 @@ [![GitHub stars](https://img.shields.io/github/stars/gojue/ecapture.svg?label=Stars&logo=github)](https://github.com/gojue/ecapture) [![GitHub forks](https://img.shields.io/github/forks/gojue/ecapture?label=Forks&logo=github)](https://github.com/gojue/ecapture) -[![CI](https://github.com/gojue/ecapture/actions/workflows/codeql-analysis.yml/badge.svg)](https://github.com/gojue/ecapture/actions/workflows/code-analysis.yml) +[![CI](https://github.com/gojue/ecapture/actions/workflows/codeql-analysis.yml/badge.svg)](https://github.com/gojue/ecapture/blob/master/.github/workflows/codeql-analysis.yml) [![Github Version](https://img.shields.io/github/v/release/gojue/ecapture?display_name=tag&include_prereleases&sort=semver)](https://github.com/gojue/ecapture/releases) -[![QQ 群](https://img.shields.io/badge/QQ群-%2312B7F5?logo=tencent-qq&logoColor=white&style=flat-square)](https://qm.qq.com/cgi-bin/qm/qr?k=iCu561fq4zdbHVdntQLFV0Xugrnf7Hpv&jump_from=webapi&authKey=YamGv189Cg+KFdQt1Qnsw6GZlpx8BYA+G2WZFezohY4M03V+l0eElZWOhZj/wR/5) +[![Docker](https://img.shields.io/docker/pulls/gojue/ecapture?style=flat&logo=docker)](https://github.com/gojue/ecapture) +[![QQ 群](https://img.shields.io/badge/QQ_群-%2312B7F5?logo=tencent-qq&logoColor=white&style=flat)](https://qm.qq.com/cgi-bin/qm/qr?k=iCu561fq4zdbHVdntQLFV0Xugrnf7Hpv&jump_from=webapi&authKey=YamGv189Cg+KFdQt1Qnsw6GZlpx8BYA+G2WZFezohY4M03V+l0eElZWOhZj/wR/5) ### eCapture(旁观者): 基于eBPF技术实现SSL/TLS加密的明文捕获，无需CA证书。 -> [!TIP] -> 支持Linux系统内核x86_64 4.18及以上版本，aarch64 5.5及以上版本； -> 需要ROOT权限； -> 不支持Windows、macOS系统； +> [!IMPORTANT] +> 支持Linux/Android系统内核x86_64 4.18及以上版本，aarch64 5.5及以上版本；不支持Windows、macOS系统；需要ROOT权限； ---- @@ -47,7 +46,7 @@ eCapture的中文名字为**旁观者**，即「**当局者迷，旁观者清** ### ELF可执行文件 -> [!IMPORTANT] +> [!TIP] > 支持 Linux/Android的x86_64/aarch64 CPU架构。 下载 [release](https://github.com/gojue/ecapture/releases) 的二进制包，可直接使用。 @@ -61,7 +60,7 @@ eCapture的中文名字为**旁观者**，即「**当局者迷，旁观者清** # 拉取镜像 docker pull gojue/ecapture:latest # 运行 -docker run --rm --privileged=true --net=host -v ${宿主机文件路径}:${容器内路径} gojue/ecapture ARGS +docker run --rm --privileged=true --net=host -v ${宿主机文件路径}:${容器内路径} gojue/ecapture ${启动参数} ``` # 小试身手 @@ -266,7 +265,9 @@ eCapture 还支持其他模块，如`bash`、`mysql`、`nss`、`postgres`等， # 贡献 -参考 [CONTRIBUTING](./CONTRIBUTING.md)的介绍，提交缺陷、补丁、建议等，非常感谢。 +参考 [CONTRIBUTING](./CONTRIBUTING.md) +的介绍，提交缺陷、补丁、建议等，非常感谢。另外，[eCapture路线图](https://github.com/orgs/gojue/projects/1) +里列出了很多待开发的任务，也欢迎您一起建设。 # 编译 From e71d2ede99a73927a9392704761bcd2690f80f2c Mon Sep 17 00:00:00 2001 From: CFC4N Date: Mon, 7 Oct 2024 10:55:13 +0800 Subject: [PATCH 7/8] pkg: fix errorcheck. Signed-off-by: CFC4N --- pkg/har/ctx.go | 6 +++--- pkg/har/har_handlers.go | 4 ++-- pkg/har/har_handlers_test.go | 1 + pkg/har/har_test.go | 7 +++++-- pkg/messageview/messageview.go | 14 +++++++------- pkg/messageview/messageview_test.go | 24 ++++++++++++------------ 6 files changed, 30 insertions(+), 26 deletions(-) diff --git a/pkg/har/ctx.go b/pkg/har/ctx.go index 6c0663313..36dfc305c 100644 --- a/pkg/har/ctx.go +++ b/pkg/har/ctx.go @@ -61,12 +61,12 @@ func TestContext(req *http.Request) (func(), error) { ctxmu.Lock() defer ctxmu.Unlock() - ctx, ok := ctxs[req] + _, ok := ctxs[req] if ok { return func() { unlink(req) }, nil } - var err error - ctx, err = genID() + + ctx, err := genID() if err != nil { return nil, err } diff --git a/pkg/har/har_handlers.go b/pkg/har/har_handlers.go index 91f9e17eb..19e7ae51d 100644 --- a/pkg/har/har_handlers.go +++ b/pkg/har/har_handlers.go @@ -56,7 +56,7 @@ func (h *exportHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { rw.Header().Set("Content-Type", "application/json; charset=utf-8") hl := h.logger.Export() - json.NewEncoder(rw).Encode(hl) + _ = json.NewEncoder(rw).Encode(hl) } // ServeHTTP resets the log, which clears its entries. @@ -79,7 +79,7 @@ func (h *resetHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { if v { rw.Header().Set("Content-Type", "application/json; charset=utf-8") hl := h.logger.ExportAndReset() - json.NewEncoder(rw).Encode(hl) + _ = json.NewEncoder(rw).Encode(hl) } else { h.logger.Reset() rw.WriteHeader(http.StatusNoContent) diff --git a/pkg/har/har_handlers_test.go b/pkg/har/har_handlers_test.go index 30690d9e2..294428c45 100644 --- a/pkg/har/har_handlers_test.go +++ b/pkg/har/har_handlers_test.go @@ -29,6 +29,7 @@ func TestExportHandlerServeHTTP(t *testing.T) { t.Fatalf("http.NewRequest(): got %v, want no error", err) } + t.Logf("req host:%s", req.Host) //if err := logger.ModifyRequest(req); err != nil { // t.Fatalf("ModifyRequest(): got %v, want no error", err) //} diff --git a/pkg/har/har_test.go b/pkg/har/har_test.go index e42a3e275..633409005 100644 --- a/pkg/har/har_test.go +++ b/pkg/har/har_test.go @@ -329,17 +329,20 @@ func TestModifyRequestBodyMultipart(t *testing.T) { body := new(bytes.Buffer) mpw := multipart.NewWriter(body) - mpw.SetBoundary("boundary") + _ = mpw.SetBoundary("boundary") if err := mpw.WriteField("key", "value"); err != nil { t.Errorf("mpw.WriteField(): got %v, want no error", err) } w, err := mpw.CreateFormFile("file", "test.txt") + if err != nil { + t.Fatalf("create file failed:%v", err) + } if _, err = w.Write([]byte("file contents")); err != nil { t.Fatalf("Write(): got %v, want no error", err) } - mpw.Close() + _ = mpw.Close() req, err := http.NewRequest("POST", "http://example.com", body) if err != nil { diff --git a/pkg/messageview/messageview.go b/pkg/messageview/messageview.go index 8d2e6e383..89da0e9d9 100644 --- a/pkg/messageview/messageview.go +++ b/pkg/messageview/messageview.go @@ -93,7 +93,7 @@ func (mv *MessageView) SnapshotRequest(req *http.Request) error { mv.compress = req.Header.Get("Content-Encoding") - req.Header.WriteSubset(buf, map[string]bool{ + _ = req.Header.WriteSubset(buf, map[string]bool{ "Host": true, "Content-Length": true, "Transfer-Encoding": true, @@ -118,7 +118,7 @@ func (mv *MessageView) SnapshotRequest(req *http.Request) error { if mv.chunked { cw := httputil.NewChunkedWriter(buf) - cw.Write(data) + _, _ = cw.Write(data) cw.Close() } else { buf.Write(data) @@ -129,7 +129,7 @@ func (mv *MessageView) SnapshotRequest(req *http.Request) error { req.Body = io.NopCloser(bytes.NewReader(data)) if req.Trailer != nil { - req.Trailer.Write(buf) + _ = req.Trailer.Write(buf) } else if mv.chunked { fmt.Fprint(buf, "\r\n") } @@ -161,7 +161,7 @@ func (mv *MessageView) SnapshotResponse(res *http.Response) error { mv.compress = "" } - res.Header.WriteSubset(buf, map[string]bool{ + _ = res.Header.WriteSubset(buf, map[string]bool{ "Content-Length": true, "Transfer-Encoding": true, }) @@ -181,12 +181,12 @@ func (mv *MessageView) SnapshotResponse(res *http.Response) error { if err != nil { return err } - res.Body.Close() + _ = res.Body.Close() if mv.chunked { cw := httputil.NewChunkedWriter(buf) cw.Write(data) - cw.Close() + _ = cw.Close() } else { buf.Write(data) } @@ -196,7 +196,7 @@ func (mv *MessageView) SnapshotResponse(res *http.Response) error { res.Body = io.NopCloser(bytes.NewReader(data)) if res.Trailer != nil { - res.Trailer.Write(buf) + _ = res.Trailer.Write(buf) } else if mv.chunked { fmt.Fprint(buf, "\r\n") } diff --git a/pkg/messageview/messageview_test.go b/pkg/messageview/messageview_test.go index 14b9f288a..2e973e52a 100644 --- a/pkg/messageview/messageview_test.go +++ b/pkg/messageview/messageview_test.go @@ -265,9 +265,9 @@ func TestRequestViewChunkedTransferEncoding(t *testing.T) { func TestRequestViewDecodeGzipContentEncoding(t *testing.T) { body := new(bytes.Buffer) gw := gzip.NewWriter(body) - gw.Write([]byte("body content")) - gw.Flush() - gw.Close() + _, _ = gw.Write([]byte("body content")) + _ = gw.Flush() + _ = gw.Close() req, err := http.NewRequest("GET", "http://example.com/path?k=v", body) if err != nil { @@ -331,9 +331,9 @@ func TestRequestViewDecodeDeflateContentEncoding(t *testing.T) { if err != nil { t.Fatalf("flate.NewWriter(): got %v, want no error", err) } - dw.Write([]byte("body content")) - dw.Flush() - dw.Close() + _, _ = dw.Write([]byte("body content")) + _ = dw.Flush() + _ = dw.Close() req, err := http.NewRequest("GET", "http://example.com/path?k=v", body) if err != nil { @@ -614,9 +614,9 @@ func TestResponseViewChunkedTransferEncoding(t *testing.T) { func TestResponseViewDecodeGzipContentEncoding(t *testing.T) { body := new(bytes.Buffer) gw := gzip.NewWriter(body) - gw.Write([]byte("body content")) - gw.Flush() - gw.Close() + _, _ = gw.Write([]byte("body content")) + _ = gw.Flush() + _ = gw.Close() res := proxy.NewResponse(200, body, nil) res.TransferEncoding = []string{"chunked"} @@ -700,9 +700,9 @@ func TestResponseViewDecodeDeflateContentEncoding(t *testing.T) { if err != nil { t.Fatalf("flate.NewWriter(): got %v, want no error", err) } - dw.Write([]byte("body content")) - dw.Flush() - dw.Close() + _, _ = dw.Write([]byte("body content")) + _ = dw.Flush() + _ = dw.Close() res := proxy.NewResponse(200, body, nil) res.TransferEncoding = []string{"chunked"} From 7d2d288322caf7f65698621a51468bd07726e67e Mon Sep 17 00:00:00 2001 From: CFC4N Date: Mon, 7 Oct 2024 11:07:30 +0800 Subject: [PATCH 8/8] pkg: fix errorcheck. Signed-off-by: CFC4N --- pkg/messageview/messageview.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/messageview/messageview.go b/pkg/messageview/messageview.go index 89da0e9d9..dc3474da4 100644 --- a/pkg/messageview/messageview.go +++ b/pkg/messageview/messageview.go @@ -185,7 +185,7 @@ func (mv *MessageView) SnapshotResponse(res *http.Response) error { if mv.chunked { cw := httputil.NewChunkedWriter(buf) - cw.Write(data) + _, _ = cw.Write(data) _ = cw.Close() } else { buf.Write(data)