Performance impact of publicsuffix parsing

[vanbroup]

The weppos publicsuffix package has a significant impact on the performance of zcrypto/x509 and zmap/zlint, https://github.com/globalsign/publicsuffix might help to improve performance significantly.

See the pprof snippet below where the publicsuffix.ParseFromListWithOptions call takes about 35% of the whole runtime.

See also:

• complete pprof diagram zcrypto-zmap-pprof.pdf
• issue Performance needs some work weppos/publicsuffix-go#133.
• x509_modified.go#L1484

(pprof) list ParseFromListWithOptions
Total: 1.78s
ROUTINE ======================== github.com/weppos/publicsuffix-go/publicsuffix.ParseFromListWithOptions in /home/digitorus/go/src/github.com/weppos/publicsuffix-go/publicsuffix/publicsuffix.go
         0      620ms (flat, cum) 34.83% of Total
         .          .    453:   n, err := normalize(name)
         .          .    454:   if err != nil {
         .          .    455:           return nil, err
         .          .    456:   }
         .          .    457:
         .      460ms    458:   r := l.Find(n, options)
         .       90ms    459:   if tld := r.Decompose(n)[1]; tld == "" {
         .          .    460:           return nil, fmt.Errorf("%s is a suffix", n)
         .          .    461:   }
         .          .    462:
         .          .    463:   dn := &DomainName{Rule: r}
         .       70ms    464:   dn.TLD, dn.SLD, dn.TRD = decompose(r, n)
         .          .    465:   return dn, nil
         .          .    466:}
         .          .    467:
         .          .    468:func normalize(name string) (string, error) {
         .          .    469:   ret := strings.ToLower(name)

Used test for profiling:
go test -bench=. -benchmem -cpuprofile profile.out

package main

import (
	"encoding/pem"
	"testing"

	"github.com/zmap/zcrypto/x509"
	"github.com/zmap/zlint"
)

const cert = `-----BEGIN CERTIFICATE-----
MIIF+TCCBOGgAwIBAgISAyn7mY0eJNhX/0WOSGr4+2i0MA0GCSqGSIb3DQEBCwUA
MEoxCzAJBgNVBAYTAlVTMRYwFAYDVQQKEw1MZXQncyBFbmNyeXB0MSMwIQYDVQQD
ExpMZXQncyBFbmNyeXB0IEF1dGhvcml0eSBYMzAeFw0xODA3MTcxNDIwMDlaFw0x
ODEwMTUxNDIwMDlaMBIxEDAOBgNVBAMTB3ptYXAuaW8wggEiMA0GCSqGSIb3DQEB
AQUAA4IBDwAwggEKAoIBAQDDyEPCKn+SDvBD/u5dLL0DIp+ApmcnpDActPJPq3iW
y21PZNDy4IYjfXw77r54D473Vic/l/SyVGw6XGKmjtaup2Z/70ZDDq0PCXLudQdZ
kWop/MC2KWFSUOtJC48XAxAR9sCK5TyWqme+epXKp6zId7zjE95gIiqyh8THNj1Y
0G819h0fAesiKrSJws8I0eUCseAReQMk6IUJNNU61tf+cni5XxEMYg8Ek0u/RjCE
H9b8OcXrb+3+wuaOtd4oQsgS+LHcDfFkioGogDlG1t49bDUFRI3FRNtPEwNker4z
ywtYGONJvJC0Pn5FtqZE8JhuCdxeRRk46ZobYnHJFODRAgMBAAGjggMPMIIDCzAO
BgNVHQ8BAf8EBAMCBaAwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMAwG
A1UdEwEB/wQCMAAwHQYDVR0OBBYEFNFrSPZACAq+QmNXUllxRmJjPlPeMB8GA1Ud
IwQYMBaAFKhKamMEfd265tE5t6ZFZe/zqOyhMG8GCCsGAQUFBwEBBGMwYTAuBggr
BgEFBQcwAYYiaHR0cDovL29jc3AuaW50LXgzLmxldHNlbmNyeXB0Lm9yZzAvBggr
BgEFBQcwAoYjaHR0cDovL2NlcnQuaW50LXgzLmxldHNlbmNyeXB0Lm9yZy8wEgYD
VR0RBAswCYIHem1hcC5pbzCB/gYDVR0gBIH2MIHzMAgGBmeBDAECATCB5gYLKwYB
BAGC3xMBAQEwgdYwJgYIKwYBBQUHAgEWGmh0dHA6Ly9jcHMubGV0c2VuY3J5cHQu
b3JnMIGrBggrBgEFBQcCAjCBngyBm1RoaXMgQ2VydGlmaWNhdGUgbWF5IG9ubHkg
YmUgcmVsaWVkIHVwb24gYnkgUmVseWluZyBQYXJ0aWVzIGFuZCBvbmx5IGluIGFj
Y29yZGFuY2Ugd2l0aCB0aGUgQ2VydGlmaWNhdGUgUG9saWN5IGZvdW5kIGF0IGh0
dHBzOi8vbGV0c2VuY3J5cHQub3JnL3JlcG9zaXRvcnkvMIIBBAYKKwYBBAHWeQIE
AgSB9QSB8gDwAHYAKTxRllTIOWW6qlD8WAfUt2+/WHopctykwwz05UVH9HgAAAFk
qNPIygAABAMARzBFAiEAiBDl2llqZ+NAx+qKDR4Nz7skZFyXBCIIleZvqTEfOtYC
IAPJQoWeSH9dTrIamnoWMz0GcU5pmFiThRvV7lYzBjxFAHYAVYHUwhaQNgFK6gub
VzxT8MDkOHhwJQgXL6OqHQcT0wwAAAFkqNPKdQAABAMARzBFAiEA9cuUQMkCFTQc
T/c+xJSufZy4sCLBwVAkeaSqn/gRVX4CIFaMT0Y/HDP92CbcnioU42ycKC2Rp4av
IYylOSyBvA4sMA0GCSqGSIb3DQEBCwUAA4IBAQCDx1Tjl+IErHLZh3GqTINsbKg5
9UuwChKxoZIfaONl+Ot7pZwiMJeUrrE/1IJyPvItfpmCzKrucJEt5Y0iqWiC/kYt
GGA1qt1JyqYCXuQ5EAP7PRW4OrtfRXPZ8A4TEttU5NPw0bEbHde030soHiwvFSvn
NxN5jkMRHMVT6h5B9JcF9HAM0RRCfd8GKigbqrJkC6DJwQ+4VBU/e6JwQSti0S4T
YR/hQaPp410CUKxYrN68bI2HApw2qahzmIVCeo3gwBpickZsf+J+z9+VFlSHivVu
tuzpSShRCSSym392PUfxgkvfxy97+2pveJ45uAv7z/pfWIwYVUfgHktN92Bz
-----END CERTIFICATE-----`

func BenchmarkHandler(b *testing.B) {
	block, _ := pem.Decode([]byte(cert))

	for n := 0; n < b.N; n++ {
		parsed, err := x509.ParseCertificate(block.Bytes)
		if err != nil {
			b.Fatal(err)
		}
		zlint.LintCertificate(parsed)
	}
}

cc: @szank, @zakird

[vanbroup]

PR weppos/publicsuffix-go#143 has drastically improved the performance of this dependency, please consider this as closed.