From e200f4e84f8d2e4c762cb8fa389b0cfde11bb984 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathias=20Sch=C3=BCpany?= Date: Sat, 25 Jun 2022 11:40:16 +0200 Subject: [PATCH 1/4] Fix certificate NotBefore field in json output (#33) --- pkg/tlsx/tls/tls.go | 2 +- pkg/tlsx/ztls/ztls.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/tlsx/tls/tls.go b/pkg/tlsx/tls/tls.go index 7380ec48..75a4489c 100644 --- a/pkg/tlsx/tls/tls.go +++ b/pkg/tlsx/tls/tls.go @@ -169,7 +169,7 @@ func convertCertificateToResponse(cert *x509.Certificate) clients.CertificateRes response := clients.CertificateResponse{ SubjectAN: cert.DNSNames, Emails: cert.EmailAddresses, - NotBefore: cert.NotAfter, + NotBefore: cert.NotBefore, NotAfter: cert.NotAfter, Expired: clients.IsExpired(cert.NotAfter), SelfSigned: clients.IsSelfSigned(cert.AuthorityKeyId, cert.SubjectKeyId), diff --git a/pkg/tlsx/ztls/ztls.go b/pkg/tlsx/ztls/ztls.go index 90e9faf8..aa906f4d 100644 --- a/pkg/tlsx/ztls/ztls.go +++ b/pkg/tlsx/ztls/ztls.go @@ -189,7 +189,7 @@ func convertCertificateToResponse(cert *x509.Certificate) clients.CertificateRes return clients.CertificateResponse{ SubjectAN: cert.DNSNames, Emails: cert.EmailAddresses, - NotBefore: cert.NotAfter, + NotBefore: cert.NotBefore, NotAfter: cert.NotAfter, Expired: clients.IsExpired(cert.NotAfter), SelfSigned: clients.IsSelfSigned(cert.AuthorityKeyId, cert.SubjectKeyId), From 157a8b913597a37ec45ba8b2dd302f6675386a84 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Jun 2022 13:22:05 +0530 Subject: [PATCH 2/4] chore(deps): bump alpine from 3.15.4 to 3.16.0 (#35) Bumps alpine from 3.15.4 to 3.16.0. --- updated-dependencies: - dependency-name: alpine dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 3a40d336..957b0f6e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM golang:1.18.2-alpine3.14 AS build-env RUN apk add --no-cache build-base RUN go install -v github.com/projectdiscovery/tlsx/cmd/tlsx@latest -FROM alpine:3.15.4 +FROM alpine:3.16.0 RUN apk add --no-cache bind-tools ca-certificates COPY --from=build-env /go/bin/tlsx /usr/local/bin/tlsx ENTRYPOINT ["tlsx"] From 957b30b5c267e7afa673757cb8b01e800655a176 Mon Sep 17 00:00:00 2001 From: Ice3man Date: Wed, 29 Jun 2022 15:44:13 +0530 Subject: [PATCH 3/4] Added probe-status flag to display status and errors in JSON output (#28) * Added errors-json flag to show errors in JSON * workflow fixes * Added misc changes as per review * misc flag updates Co-authored-by: sandeep --- .github/workflows/sonarcloud.yml | 6 ++--- README.md | 19 +++++++------- cmd/tlsx/main.go | 3 ++- go.mod | 6 ++--- internal/runner/runner.go | 1 - pkg/output/output.go | 10 +++++++ pkg/tlsx/clients/clients.go | 45 ++++++++++++++++++-------------- pkg/tlsx/tls/tls.go | 8 +++--- pkg/tlsx/tlsx.go | 6 ++++- pkg/tlsx/ztls/ztls.go | 10 ++++--- 10 files changed, 70 insertions(+), 44 deletions(-) diff --git a/.github/workflows/sonarcloud.yml b/.github/workflows/sonarcloud.yml index e0df6963..20e7e48f 100644 --- a/.github/workflows/sonarcloud.yml +++ b/.github/workflows/sonarcloud.yml @@ -27,9 +27,9 @@ jobs: go test -coverprofile=./cov.out ./... - name: Run Gosec Security Scanner - run: | - go install github.com/securego/gosec/cmd/gosec@latest - gosec -no-fail -fmt=sonarqube -out report.json ./... + uses: securego/gosec@master + with: + args: '-no-fail -fmt=sonarqube -out report.json ./...' - name: SonarCloud Scan uses: SonarSource/sonarcloud-github-action@master diff --git a/README.md b/README.md index 6c129d77..594bdef4 100644 --- a/README.md +++ b/README.md @@ -70,14 +70,15 @@ SCAN-MODE: -ps, -pre-handshake enable pre-handshake tls connection (early termination) using ztls PROBES: - -san display subject alternative names - -cn display subject common names - -so display subject organization name - -tv, -tls-version display used tls version - -cipher display used cipher - -ex, -expired display validity status of certificate - -ss, -self-signed display status of self-signed certificate - -hash string display certificate fingerprint hashes (md5,sha1,sha256) + -san display subject alternative names + -cn display subject common names + -so display subject organization name + -tv, -tls-version display used tls version + -cipher display used cipher + -ex, -expired display validity status of certificate + -ss, -self-signed display status of self-signed certificate + -hash string display certificate fingerprint hashes (md5,sha1,sha256) + -tps, -probe-status display tls probe status CONFIGURATIONS: -config string path to the tlsx configuration file @@ -88,7 +89,7 @@ CONFIGURATIONS: -min-version string minimum tls version to accept (ssl30,tls10,tls11,tls12,tls13) -max-version string maximum tls version to accept (ssl30,tls10,tls11,tls12,tls13) -tc, -tls-chain display tls chain in json output - -verify-cert enable verification of server certificate + -vc, -verify-cert enable verification of server certificate OPTIMIZATIONS: -c, -concurrency int number of concurrent threads to process (default 300) diff --git a/cmd/tlsx/main.go b/cmd/tlsx/main.go index c92be693..ccf887de 100644 --- a/cmd/tlsx/main.go +++ b/cmd/tlsx/main.go @@ -63,6 +63,7 @@ func readFlags() error { flagSet.BoolVarP(&options.Expired, "expired", "ex", false, "display validity status of certificate"), flagSet.BoolVarP(&options.SelfSigned, "self-signed", "ss", false, "display status of self-signed certificate"), flagSet.StringVar(&options.Hash, "hash", "", "display certificate fingerprint hashes (md5,sha1,sha256)"), + flagSet.BoolVarP(&options.ProbeStatus, "probe-status", "tps", false, "display tls probe status"), ) flagSet.CreateGroup("configs", "Configurations", @@ -74,7 +75,7 @@ func readFlags() error { flagSet.StringVar(&options.MinVersion, "min-version", "", "minimum tls version to accept (ssl30,tls10,tls11,tls12,tls13)"), flagSet.StringVar(&options.MaxVersion, "max-version", "", "maximum tls version to accept (ssl30,tls10,tls11,tls12,tls13)"), flagSet.BoolVarP(&options.TLSChain, "tls-chain", "tc", false, "display tls chain in json output"), - flagSet.BoolVar(&options.VerifyServerCertificate, "verify-cert", false, "enable verification of server certificate"), + flagSet.BoolVarP(&options.VerifyServerCertificate, "verify-cert", "vc", false, "enable verification of server certificate"), ) flagSet.CreateGroup("optimizations", "OPTIMIZATIONS", diff --git a/go.mod b/go.mod index 73b6d2fe..52fd2eff 100644 --- a/go.mod +++ b/go.mod @@ -4,12 +4,15 @@ go 1.17 require ( github.com/json-iterator/go v1.1.12 + github.com/logrusorgru/aurora v2.0.3+incompatible github.com/pkg/errors v0.9.1 github.com/projectdiscovery/fastdialer v0.0.16-0.20220620143737-2ba20b53770a github.com/projectdiscovery/fileutil v0.0.0-20220506114156-c4ab20801483 github.com/projectdiscovery/goflags v0.0.8 github.com/projectdiscovery/gologger v1.1.4 + github.com/projectdiscovery/iputil v0.0.0-20220613112553-9b6873b2c619 github.com/projectdiscovery/mapcidr v1.0.0 + github.com/rs/xid v1.4.0 github.com/zmap/zcrypto v0.0.0-20220605182715-4dfcec6e9a8c ) @@ -20,19 +23,16 @@ require ( github.com/dimchansky/utfbom v1.1.1 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/karrick/godirwalk v1.16.1 // indirect - github.com/logrusorgru/aurora v2.0.3+incompatible // indirect github.com/miekg/dns v1.1.43 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e // indirect github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 // indirect github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa // indirect - github.com/projectdiscovery/iputil v0.0.0-20220613112553-9b6873b2c619 // indirect github.com/projectdiscovery/networkpolicy v0.0.1 // indirect github.com/projectdiscovery/retryabledns v1.0.13-0.20210916165024-76c5b76fd59a // indirect github.com/projectdiscovery/retryablehttp-go v1.0.2 // indirect github.com/projectdiscovery/stringsutil v0.0.0-20220422150559-b54fb5dc6833 // indirect - github.com/rs/xid v1.4.0 // indirect github.com/syndtr/goleveldb v1.0.0 // indirect github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6 // indirect github.com/weppos/publicsuffix-go v0.15.1-0.20220329081811-9a40b608a236 // indirect diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 1128a0aa..c8c2780b 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -131,7 +131,6 @@ func (r *Runner) processInputElementWorker(inputs chan taskInput, wg *sync.WaitG response, err := r.tlsxService.Connect(task.host, task.port) if err != nil { gologger.Warning().Msgf("Could not connect input %s: %s", task.Address(), err) - continue } if response != nil { if err := r.outputWriter.Write(response); err != nil { diff --git a/pkg/output/output.go b/pkg/output/output.go index 92cdb8d1..2b96f701 100644 --- a/pkg/output/output.go +++ b/pkg/output/output.go @@ -137,6 +137,16 @@ func (w *StandardWriter) formatStandard(output *clients.Response) ([]byte, error if !w.options.SAN && !w.options.CN { builder.WriteString(outputPrefix) } + if !output.ProbeStatus { + builder.WriteString(" [") + builder.WriteString(w.aurora.Red("failed").String()) + builder.WriteString("]") + } + if w.options.ProbeStatus && output.ProbeStatus { + builder.WriteString(" [") + builder.WriteString(w.aurora.Green("success").String()) + builder.WriteString("]") + } if w.options.SO && len(cert.SubjectOrg) > 0 { builder.WriteString(" [") builder.WriteString(w.aurora.BrightYellow(strings.Join(cert.SubjectOrg, ",")).String()) diff --git a/pkg/tlsx/clients/clients.go b/pkg/tlsx/clients/clients.go index a8ff6bde..95bafd4e 100644 --- a/pkg/tlsx/clients/clients.go +++ b/pkg/tlsx/clients/clients.go @@ -37,6 +37,8 @@ type Options struct { JSON bool // TLSChain enables printing TLS chain information to output TLSChain bool + // ProbeStatus enables writing of errors with json output + ProbeStatus bool // CertsOnly enables early SSL termination using ztls flag CertsOnly bool // RespOnly displays TLS respones only in CLI output @@ -92,24 +94,29 @@ type Options struct { // Response is the response returned for a TLS grab event type Response struct { // Timestamp is the timestamp for certificate response - Timestamp time.Time `json:"timestamp,omitempty"` + Timestamp *time.Time `json:"timestamp,omitempty"` // Host is the host to make request to Host string `json:"host"` // IP is the IP address the request was made to IP string `json:"ip,omitempty"` // Port is the port to make request to Port string `json:"port"` + // ProbeStatus is false if the tls probe failed + ProbeStatus bool `json:"probe_status"` + // Error is the optional error for tls request included + // with errors_json flag. + Error string `json:"error,omitempty"` // Version is the tls version responded by the server - Version string `json:"tls-version"` + Version string `json:"tls_version,omitempty"` // Cipher is the cipher for the tls request Cipher string `json:"cipher,omitempty"` // CertificateResponse is the leaf certificate embedded in json - CertificateResponse `json:",inline"` + *CertificateResponse `json:",inline"` // TLSConnection is the client used for TLS connection // when ran using scan-mode auto. - TLSConnection string `json:"tls-connection,omitempty"` + TLSConnection string `json:"tls_connection,omitempty"` // Chain is the chain of certificates - Chain []CertificateResponse `json:"chain,omitempty"` + Chain []*CertificateResponse `json:"chain,omitempty"` } // CertificateResponse is the response for a certificate @@ -117,40 +124,40 @@ type CertificateResponse struct { // Expired specifies whether the certificate has expired Expired bool `json:"expired,omitempty"` // SelfSigned returns true if the certificate is self-signed - SelfSigned bool `json:"self-signed,omitempty"` + SelfSigned bool `json:"self_signed,omitempty"` // NotBefore is the not-before time for certificate - NotBefore time.Time `json:"not-before,omitempty"` + NotBefore time.Time `json:"not_before,omitempty"` // NotAfter is the not-after time for certificate - NotAfter time.Time `json:"not-after,omitempty"` + NotAfter time.Time `json:"not_after,omitempty"` // SubjectDN is the distinguished name for cert - SubjectDN string `json:"subject-dn,omitempty"` + SubjectDN string `json:"subject_dn,omitempty"` // SubjectCN is the common name for cert - SubjectCN string `json:"subject-cn,omitempty"` + SubjectCN string `json:"subject_cn,omitempty"` // SubjectOrg is the organization for cert subject - SubjectOrg []string `json:"subject-org,omitempty"` + SubjectOrg []string `json:"subject_org,omitempty"` // SubjectAN is a list of Subject Alternative Names for the certificate - SubjectAN []string `json:"subject-an,omitempty"` + SubjectAN []string `json:"subject_an,omitempty"` // IssuerDN is the distinguished name for cert - IssuerDN string `json:"issuer-dn,omitempty"` + IssuerDN string `json:"issuer_dn,omitempty"` // IssuerCN is the common name for cert - IssuerCN string `json:"issuer-cn,omitempty"` + IssuerCN string `json:"issuer_cn,omitempty"` // IssuerOrg is the organization for cert issuer - IssuerOrg []string `json:"issuer-org,omitempty"` + IssuerOrg []string `json:"issuer_org,omitempty"` // Emails is a list of Emails for the certificate Emails []string `json:"emails,omitempty"` // FingerprintHash is the hashes for certificate - FingerprintHash CertificateResponseFingerprintHash `json:"fingerprint-hash"` + FingerprintHash CertificateResponseFingerprintHash `json:"fingerprint_hash,omitempty"` } // CertificateDistinguishedName is a distinguished certificate name type CertificateDistinguishedName struct { Country []string `json:"country,omitempty"` Organization []string `json:"organization,omitempty"` - OrganizationalUnit []string `json:"organizational-unit,omitempty"` + OrganizationalUnit []string `json:"organizational_unit,omitempty"` Locality []string `json:"locality,omitempty"` Province []string `json:"province,omitempty"` - StreetAddress []string `json:"street-address,omitempty"` - CommonName string `json:"common-name,omitempty"` + StreetAddress []string `json:"street_address,omitempty"` + CommonName string `json:"common_name,omitempty"` } // CertificateResponseFingerprintHash is a response for fingerprint hash of cert diff --git a/pkg/tlsx/tls/tls.go b/pkg/tlsx/tls/tls.go index 75a4489c..2f9d801f 100644 --- a/pkg/tlsx/tls/tls.go +++ b/pkg/tlsx/tls/tls.go @@ -147,10 +147,12 @@ func (c *Client) Connect(hostname, port string) (*clients.Response, error) { leafCertificate := connectionState.PeerCertificates[0] certificateChain := connectionState.PeerCertificates[1:] + now := time.Now() response := &clients.Response{ - Timestamp: time.Now(), + Timestamp: &now, Host: hostname, IP: resolvedIP, + ProbeStatus: true, Port: port, Version: tlsVersion, Cipher: tlsCipher, @@ -165,8 +167,8 @@ func (c *Client) Connect(hostname, port string) (*clients.Response, error) { return response, nil } -func convertCertificateToResponse(cert *x509.Certificate) clients.CertificateResponse { - response := clients.CertificateResponse{ +func convertCertificateToResponse(cert *x509.Certificate) *clients.CertificateResponse { + response := &clients.CertificateResponse{ SubjectAN: cert.DNSNames, Emails: cert.EmailAddresses, NotBefore: cert.NotBefore, diff --git a/pkg/tlsx/tlsx.go b/pkg/tlsx/tlsx.go index 028fc623..95e2bce8 100644 --- a/pkg/tlsx/tlsx.go +++ b/pkg/tlsx/tlsx.go @@ -41,7 +41,11 @@ func New(options *clients.Options) (*Service, error) { func (s *Service) Connect(host, port string) (*clients.Response, error) { resp, err := s.client.Connect(host, port) if err != nil { - return nil, errors.Wrap(err, "could not connect to host") + wrappedErr := errors.Wrap(err, "could not connect to host") + if s.options.ProbeStatus { + return &clients.Response{Host: host, Port: port, Error: err.Error(), ProbeStatus: false}, wrappedErr + } + return nil, wrappedErr } return resp, nil } diff --git a/pkg/tlsx/ztls/ztls.go b/pkg/tlsx/ztls/ztls.go index aa906f4d..0c8771b7 100644 --- a/pkg/tlsx/ztls/ztls.go +++ b/pkg/tlsx/ztls/ztls.go @@ -159,10 +159,12 @@ func (c *Client) Connect(hostname, port string) (*clients.Response, error) { tlsVersion := versionToTLSVersionString[uint16(hl.ServerHello.Version)] tlsCipher := hl.ServerHello.CipherSuite.String() + now := time.Now() response := &clients.Response{ - Timestamp: time.Now(), + Timestamp: &now, Host: hostname, IP: resolvedIP, + ProbeStatus: true, Port: port, Version: tlsVersion, Cipher: tlsCipher, @@ -182,11 +184,11 @@ func parseSimpleTLSCertificate(cert tls.SimpleCertificate) *x509.Certificate { return parsed } -func convertCertificateToResponse(cert *x509.Certificate) clients.CertificateResponse { +func convertCertificateToResponse(cert *x509.Certificate) *clients.CertificateResponse { if cert == nil { - return clients.CertificateResponse{} + return nil } - return clients.CertificateResponse{ + return &clients.CertificateResponse{ SubjectAN: cert.DNSNames, Emails: cert.EmailAddresses, NotBefore: cert.NotBefore, From 2292738a79a7b64c466a646679c36c82c6f3c11f Mon Sep 17 00:00:00 2001 From: sandeep Date: Wed, 29 Jun 2022 17:05:13 +0530 Subject: [PATCH 4/4] version update --- internal/runner/banner.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/runner/banner.go b/internal/runner/banner.go index 19862b6a..d6c1c635 100644 --- a/internal/runner/banner.go +++ b/internal/runner/banner.go @@ -17,7 +17,7 @@ var banner = fmt.Sprintf(` |_| |____|___/_/\_\ %s `, version) -var version = "v0.0.1" +var version = "v0.0.2" // validateOptions validates the provided options for crawler func (r *Runner) validateOptions() error {