Merge pull request #39 from projectdiscovery/dev

Bugfix Release

.github/workflows/sonarcloud.yml

@@ -27,9 +27,9 @@ jobs:
           go test -coverprofile=./cov.out ./...
 
       - name: Run Gosec Security Scanner
-        run: |
-          go install github.com/securego/gosec/cmd/gosec@latest
-          gosec -no-fail -fmt=sonarqube -out report.json ./...
+        uses: securego/gosec@master
+        with:
+          args: '-no-fail -fmt=sonarqube -out report.json ./...'
 
       - name: SonarCloud Scan
         uses: SonarSource/sonarcloud-github-action@master

Dockerfile

@@ -2,7 +2,7 @@ FROM golang:1.18.2-alpine3.14 AS build-env
 RUN apk add --no-cache build-base
 RUN go install -v github.com/projectdiscovery/tlsx/cmd/tlsx@latest
 
-FROM alpine:3.15.4
+FROM alpine:3.16.0
 RUN apk add --no-cache bind-tools ca-certificates
 COPY --from=build-env /go/bin/tlsx /usr/local/bin/tlsx
 ENTRYPOINT ["tlsx"]

README.md

@@ -70,14 +70,15 @@ SCAN-MODE:
    -ps, -pre-handshake     enable pre-handshake tls connection (early termination) using ztls
 
 PROBES:
-   -san               display subject alternative names
-   -cn                display subject common names
-   -so                display subject organization name
-   -tv, -tls-version  display used tls version
-   -cipher            display used cipher
-   -ex, -expired      display validity status of certificate
-   -ss, -self-signed  display status of self-signed certificate
-   -hash string       display certificate fingerprint hashes (md5,sha1,sha256)
+   -san                 display subject alternative names
+   -cn                  display subject common names
+   -so                  display subject organization name
+   -tv, -tls-version    display used tls version
+   -cipher              display used cipher
+   -ex, -expired        display validity status of certificate
+   -ss, -self-signed    display status of self-signed certificate
+   -hash string         display certificate fingerprint hashes (md5,sha1,sha256)
+   -tps, -probe-status  display tls probe status
 
 CONFIGURATIONS:
    -config string               path to the tlsx configuration file
@@ -88,7 +89,7 @@ CONFIGURATIONS:
    -min-version string          minimum tls version to accept (ssl30,tls10,tls11,tls12,tls13)
    -max-version string          maximum tls version to accept (ssl30,tls10,tls11,tls12,tls13)
    -tc, -tls-chain              display tls chain in json output
-   -verify-cert                 enable verification of server certificate
+   -vc, -verify-cert            enable verification of server certificate
 
 OPTIMIZATIONS:
    -c, -concurrency int  number of concurrent threads to process (default 300)

cmd/tlsx/main.go

@@ -63,6 +63,7 @@ func readFlags() error {
 		flagSet.BoolVarP(&options.Expired, "expired", "ex", false, "display validity status of certificate"),
 		flagSet.BoolVarP(&options.SelfSigned, "self-signed", "ss", false, "display status of self-signed certificate"),
 		flagSet.StringVar(&options.Hash, "hash", "", "display certificate fingerprint hashes (md5,sha1,sha256)"),
+		flagSet.BoolVarP(&options.ProbeStatus, "probe-status", "tps", false, "display tls probe status"),
 	)
 
 	flagSet.CreateGroup("configs", "Configurations",
@@ -74,7 +75,7 @@ func readFlags() error {
 		flagSet.StringVar(&options.MinVersion, "min-version", "", "minimum tls version to accept (ssl30,tls10,tls11,tls12,tls13)"),
 		flagSet.StringVar(&options.MaxVersion, "max-version", "", "maximum tls version to accept (ssl30,tls10,tls11,tls12,tls13)"),
 		flagSet.BoolVarP(&options.TLSChain, "tls-chain", "tc", false, "display tls chain in json output"),
-		flagSet.BoolVar(&options.VerifyServerCertificate, "verify-cert", false, "enable verification of server certificate"),
+		flagSet.BoolVarP(&options.VerifyServerCertificate, "verify-cert", "vc", false, "enable verification of server certificate"),
 	)
 
 	flagSet.CreateGroup("optimizations", "OPTIMIZATIONS",

go.mod

@@ -4,12 +4,15 @@ go 1.17
 
 require (
 	github.com/json-iterator/go v1.1.12
+	github.com/logrusorgru/aurora v2.0.3+incompatible
 	github.com/pkg/errors v0.9.1
 	github.com/projectdiscovery/fastdialer v0.0.16-0.20220620143737-2ba20b53770a
 	github.com/projectdiscovery/fileutil v0.0.0-20220506114156-c4ab20801483
 	github.com/projectdiscovery/goflags v0.0.8
 	github.com/projectdiscovery/gologger v1.1.4
+	github.com/projectdiscovery/iputil v0.0.0-20220613112553-9b6873b2c619
 	github.com/projectdiscovery/mapcidr v1.0.0
+	github.com/rs/xid v1.4.0
 	github.com/zmap/zcrypto v0.0.0-20220605182715-4dfcec6e9a8c
 )
 
@@ -20,19 +23,16 @@ require (
 	github.com/dimchansky/utfbom v1.1.1 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/karrick/godirwalk v1.16.1 // indirect
-	github.com/logrusorgru/aurora v2.0.3+incompatible // indirect
 	github.com/miekg/dns v1.1.43 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e // indirect
 	github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 // indirect
 	github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa // indirect
-	github.com/projectdiscovery/iputil v0.0.0-20220613112553-9b6873b2c619 // indirect
 	github.com/projectdiscovery/networkpolicy v0.0.1 // indirect
 	github.com/projectdiscovery/retryabledns v1.0.13-0.20210916165024-76c5b76fd59a // indirect
 	github.com/projectdiscovery/retryablehttp-go v1.0.2 // indirect
 	github.com/projectdiscovery/stringsutil v0.0.0-20220422150559-b54fb5dc6833 // indirect
-	github.com/rs/xid v1.4.0 // indirect
 	github.com/syndtr/goleveldb v1.0.0 // indirect
 	github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6 // indirect
 	github.com/weppos/publicsuffix-go v0.15.1-0.20220329081811-9a40b608a236 // indirect

internal/runner/banner.go

@@ -17,7 +17,7 @@ var banner = fmt.Sprintf(`
    |_| |____|___/_/\_\	%s
 `, version)
 
-var version = "v0.0.1"
+var version = "v0.0.2"
 
 // validateOptions validates the provided options for crawler
 func (r *Runner) validateOptions() error {

internal/runner/runner.go

@@ -131,7 +131,6 @@ func (r *Runner) processInputElementWorker(inputs chan taskInput, wg *sync.WaitG
 		response, err := r.tlsxService.Connect(task.host, task.port)
 		if err != nil {
 			gologger.Warning().Msgf("Could not connect input %s: %s", task.Address(), err)
-			continue
 		}
 		if response != nil {
 			if err := r.outputWriter.Write(response); err != nil {

pkg/output/output.go

@@ -137,6 +137,16 @@ func (w *StandardWriter) formatStandard(output *clients.Response) ([]byte, error
 	if !w.options.SAN && !w.options.CN {
 		builder.WriteString(outputPrefix)
 	}
+	if !output.ProbeStatus {
+		builder.WriteString(" [")
+		builder.WriteString(w.aurora.Red("failed").String())
+		builder.WriteString("]")
+	}
+	if w.options.ProbeStatus && output.ProbeStatus {
+		builder.WriteString(" [")
+		builder.WriteString(w.aurora.Green("success").String())
+		builder.WriteString("]")
+	}
 	if w.options.SO && len(cert.SubjectOrg) > 0 {
 		builder.WriteString(" [")
 		builder.WriteString(w.aurora.BrightYellow(strings.Join(cert.SubjectOrg, ",")).String())

pkg/tlsx/clients/clients.go

@@ -37,6 +37,8 @@ type Options struct {
 	JSON bool
 	// TLSChain enables printing TLS chain information to output
 	TLSChain bool
+	// ProbeStatus enables writing of errors with json output
+	ProbeStatus bool
 	// CertsOnly enables early SSL termination using ztls flag
 	CertsOnly bool
 	// RespOnly displays TLS respones only in CLI output
@@ -92,65 +94,70 @@ type Options struct {
 // Response is the response returned for a TLS grab event
 type Response struct {
 	// Timestamp is the timestamp for certificate response
-	Timestamp time.Time `json:"timestamp,omitempty"`
+	Timestamp *time.Time `json:"timestamp,omitempty"`
 	// Host is the host to make request to
 	Host string `json:"host"`
 	// IP is the IP address the request was made to
 	IP string `json:"ip,omitempty"`
 	// Port is the port to make request to
 	Port string `json:"port"`
+	// ProbeStatus is false if the tls probe failed
+	ProbeStatus bool `json:"probe_status"`
+	// Error is the optional error for tls request included
+	// with errors_json flag.
+	Error string `json:"error,omitempty"`
 	// Version is the tls version responded by the server
-	Version string `json:"tls-version"`
+	Version string `json:"tls_version,omitempty"`
 	// Cipher is the cipher for the tls request
 	Cipher string `json:"cipher,omitempty"`
 	// CertificateResponse is the leaf certificate embedded in json
-	CertificateResponse `json:",inline"`
+	*CertificateResponse `json:",inline"`
 	// TLSConnection is the client used for TLS connection
 	// when ran using scan-mode auto.
-	TLSConnection string `json:"tls-connection,omitempty"`
+	TLSConnection string `json:"tls_connection,omitempty"`
 	// Chain is the chain of certificates
-	Chain []CertificateResponse `json:"chain,omitempty"`
+	Chain []*CertificateResponse `json:"chain,omitempty"`
 }
 
 // CertificateResponse is the response for a certificate
 type CertificateResponse struct {
 	// Expired specifies whether the certificate has expired
 	Expired bool `json:"expired,omitempty"`
 	// SelfSigned returns true if the certificate is self-signed
-	SelfSigned bool `json:"self-signed,omitempty"`
+	SelfSigned bool `json:"self_signed,omitempty"`
 	// NotBefore is the not-before time for certificate
-	NotBefore time.Time `json:"not-before,omitempty"`
+	NotBefore time.Time `json:"not_before,omitempty"`
 	// NotAfter is the not-after time for certificate
-	NotAfter time.Time `json:"not-after,omitempty"`
+	NotAfter time.Time `json:"not_after,omitempty"`
 	// SubjectDN is the distinguished name for cert
-	SubjectDN string `json:"subject-dn,omitempty"`
+	SubjectDN string `json:"subject_dn,omitempty"`
 	// SubjectCN is the common name for cert
-	SubjectCN string `json:"subject-cn,omitempty"`
+	SubjectCN string `json:"subject_cn,omitempty"`
 	// SubjectOrg is the organization for cert subject
-	SubjectOrg []string `json:"subject-org,omitempty"`
+	SubjectOrg []string `json:"subject_org,omitempty"`
 	// SubjectAN is a list of Subject Alternative Names for the certificate
-	SubjectAN []string `json:"subject-an,omitempty"`
+	SubjectAN []string `json:"subject_an,omitempty"`
 	// IssuerDN is the distinguished name for cert
-	IssuerDN string `json:"issuer-dn,omitempty"`
+	IssuerDN string `json:"issuer_dn,omitempty"`
 	// IssuerCN is the common name for cert
-	IssuerCN string `json:"issuer-cn,omitempty"`
+	IssuerCN string `json:"issuer_cn,omitempty"`
 	// IssuerOrg is the organization for cert issuer
-	IssuerOrg []string `json:"issuer-org,omitempty"`
+	IssuerOrg []string `json:"issuer_org,omitempty"`
 	// Emails is a list of Emails for the certificate
 	Emails []string `json:"emails,omitempty"`
 	// FingerprintHash is the hashes for certificate
-	FingerprintHash CertificateResponseFingerprintHash `json:"fingerprint-hash"`
+	FingerprintHash CertificateResponseFingerprintHash `json:"fingerprint_hash,omitempty"`
 }
 
 // CertificateDistinguishedName is a distinguished certificate name
 type CertificateDistinguishedName struct {
 	Country            []string `json:"country,omitempty"`
 	Organization       []string `json:"organization,omitempty"`
-	OrganizationalUnit []string `json:"organizational-unit,omitempty"`
+	OrganizationalUnit []string `json:"organizational_unit,omitempty"`
 	Locality           []string `json:"locality,omitempty"`
 	Province           []string `json:"province,omitempty"`
-	StreetAddress      []string `json:"street-address,omitempty"`
-	CommonName         string   `json:"common-name,omitempty"`
+	StreetAddress      []string `json:"street_address,omitempty"`
+	CommonName         string   `json:"common_name,omitempty"`
 }
 
 // CertificateResponseFingerprintHash is a response for fingerprint hash of cert

pkg/tlsx/tls/tls.go

@@ -147,10 +147,12 @@ func (c *Client) Connect(hostname, port string) (*clients.Response, error) {
 	leafCertificate := connectionState.PeerCertificates[0]
 	certificateChain := connectionState.PeerCertificates[1:]
 
+	now := time.Now()
 	response := &clients.Response{
-		Timestamp:           time.Now(),
+		Timestamp:           &now,
 		Host:                hostname,
 		IP:                  resolvedIP,
+		ProbeStatus:         true,
 		Port:                port,
 		Version:             tlsVersion,
 		Cipher:              tlsCipher,
@@ -165,11 +167,11 @@ func (c *Client) Connect(hostname, port string) (*clients.Response, error) {
 	return response, nil
 }
 
-func convertCertificateToResponse(cert *x509.Certificate) clients.CertificateResponse {
-	response := clients.CertificateResponse{
+func convertCertificateToResponse(cert *x509.Certificate) *clients.CertificateResponse {
+	response := &clients.CertificateResponse{
 		SubjectAN:  cert.DNSNames,
 		Emails:     cert.EmailAddresses,
-		NotBefore:  cert.NotAfter,
+		NotBefore:  cert.NotBefore,
 		NotAfter:   cert.NotAfter,
 		Expired:    clients.IsExpired(cert.NotAfter),
 		SelfSigned: clients.IsSelfSigned(cert.AuthorityKeyId, cert.SubjectKeyId),

pkg/tlsx/tlsx.go

@@ -41,7 +41,11 @@ func New(options *clients.Options) (*Service, error) {
 func (s *Service) Connect(host, port string) (*clients.Response, error) {
 	resp, err := s.client.Connect(host, port)
 	if err != nil {
-		return nil, errors.Wrap(err, "could not connect to host")
+		wrappedErr := errors.Wrap(err, "could not connect to host")
+		if s.options.ProbeStatus {
+			return &clients.Response{Host: host, Port: port, Error: err.Error(), ProbeStatus: false}, wrappedErr
+		}
+		return nil, wrappedErr
 	}
 	return resp, nil
 }

pkg/tlsx/ztls/ztls.go

@@ -159,10 +159,12 @@ func (c *Client) Connect(hostname, port string) (*clients.Response, error) {
 	tlsVersion := versionToTLSVersionString[uint16(hl.ServerHello.Version)]
 	tlsCipher := hl.ServerHello.CipherSuite.String()
 
+	now := time.Now()
 	response := &clients.Response{
-		Timestamp:           time.Now(),
+		Timestamp:           &now,
 		Host:                hostname,
 		IP:                  resolvedIP,
+		ProbeStatus:         true,
 		Port:                port,
 		Version:             tlsVersion,
 		Cipher:              tlsCipher,
@@ -182,14 +184,14 @@ func parseSimpleTLSCertificate(cert tls.SimpleCertificate) *x509.Certificate {
 	return parsed
 }
 
-func convertCertificateToResponse(cert *x509.Certificate) clients.CertificateResponse {
+func convertCertificateToResponse(cert *x509.Certificate) *clients.CertificateResponse {
 	if cert == nil {
-		return clients.CertificateResponse{}
+		return nil
 	}
-	return clients.CertificateResponse{
+	return &clients.CertificateResponse{
 		SubjectAN:  cert.DNSNames,
 		Emails:     cert.EmailAddresses,
-		NotBefore:  cert.NotAfter,
+		NotBefore:  cert.NotBefore,
 		NotAfter:   cert.NotAfter,
 		Expired:    clients.IsExpired(cert.NotAfter),
 		SelfSigned: clients.IsSelfSigned(cert.AuthorityKeyId, cert.SubjectKeyId),