dramatically reduce memory usage (#758)

Run all linters per package. It allows unloading package data when it's
processed. It dramatically reduces memory (and CPU because of GC) usage.

Relates: #337

Author: jirfag

Makefile

@@ -29,6 +29,7 @@ clean:
 test: export GOLANGCI_LINT_INSTALLED = true
 test: build
 	GL_TEST_RUN=1 time ./golangci-lint run -v
+	time go run ./cmd/golangci-lint/main.go run -v
 	GL_TEST_RUN=1 time ./golangci-lint run --fast --no-config -v --skip-dirs 'test/testdata_etc,internal/(cache|renameio|robustio)'
 	GL_TEST_RUN=1 time ./golangci-lint run --no-config -v --skip-dirs 'test/testdata_etc,internal/(cache|renameio|robustio)'
 	GL_TEST_RUN=1 time go test -v ./...

README.md

@@ -178,13 +178,13 @@ $ golangci-lint help linters
 Enabled by default linters:
 deadcode: Finds unused code [fast: true, auto-fix: false]
 errcheck: Errcheck is a program for checking for unchecked errors in go programs. These unchecked errors can be critical bugs in some cases [fast: true, auto-fix: false]
-gosimple: Linter for Go source code that specializes in simplifying a code [fast: false, auto-fix: false]
-govet (vet, vetshadow): Vet examines Go source code and reports suspicious constructs, such as Printf calls whose arguments do not align with the format string [fast: false, auto-fix: false]
+gosimple (megacheck): Linter for Go source code that specializes in simplifying a code [fast: true, auto-fix: false]
+govet (vet, vetshadow): Vet examines Go source code and reports suspicious constructs, such as Printf calls whose arguments do not align with the format string [fast: true, auto-fix: false]
 ineffassign: Detects when assignments to existing variables are not used [fast: true, auto-fix: false]
-staticcheck: Staticcheck is a go vet on steroids, applying a ton of static analysis checks [fast: false, auto-fix: false]
+staticcheck (megacheck): Staticcheck is a go vet on steroids, applying a ton of static analysis checks [fast: true, auto-fix: false]
 structcheck: Finds unused struct fields [fast: true, auto-fix: false]
 typecheck: Like the front-end of a Go compiler, parses and type-checks Go code [fast: true, auto-fix: false]
-unused: Checks Go code for unused constants, variables, functions and types [fast: false, auto-fix: false]
+unused (megacheck): Checks Go code for unused constants, variables, functions and types [fast: false, auto-fix: false]
 varcheck: Finds unused global variables and constants [fast: true, auto-fix: false]
 ```
 
@@ -194,12 +194,12 @@ and the following linters are disabled by default:
 $ golangci-lint help linters
 ...
 Disabled by default linters:
-bodyclose: checks whether HTTP response body is closed successfully [fast: false, auto-fix: false]
+bodyclose: checks whether HTTP response body is closed successfully [fast: true, auto-fix: false]
 depguard: Go linter that checks if package imports are in a list of acceptable packages [fast: true, auto-fix: false]
 dogsled: Checks assignments with too many blank identifiers (e.g. x, _, _, _, := f()) [fast: true, auto-fix: false]
 dupl: Tool for code clone detection [fast: true, auto-fix: false]
 funlen: Tool for detection of long functions [fast: true, auto-fix: false]
-gochecknoglobals: Checks that no globals are present in Go code [fast: true, auto-fix: false]
+gochecknoglobals: Tool for detection of long functions [fast: true, auto-fix: false]
 gochecknoinits: Checks that no init functions are present in Go code [fast: true, auto-fix: false]
 goconst: Finds repeated strings that could be replaced by a constant [fast: true, auto-fix: false]
 gocritic: The most opinionated Go source code linter [fast: true, auto-fix: false]
@@ -209,16 +209,16 @@ gofmt: Gofmt checks whether code was gofmt-ed. By default this tool runs with -s
 goimports: Goimports does everything that gofmt does. Additionally it checks unused imports [fast: true, auto-fix: true]
 golint: Golint differs from gofmt. Gofmt reformats Go source code, whereas golint prints out style mistakes [fast: true, auto-fix: false]
 gosec (gas): Inspects source code for security problems [fast: true, auto-fix: false]
-interfacer: Linter that suggests narrower interface types [fast: false, auto-fix: false]
+interfacer: Linter that suggests narrower interface types [fast: true, auto-fix: false]
 lll: Reports long lines [fast: true, auto-fix: false]
 maligned: Tool to detect Go structs that would take less memory if their fields were sorted [fast: true, auto-fix: false]
 misspell: Finds commonly misspelled English words in comments [fast: true, auto-fix: true]
 nakedret: Finds naked returns in functions greater than a specified function length [fast: true, auto-fix: false]
 prealloc: Finds slice declarations that could potentially be preallocated [fast: true, auto-fix: false]
 scopelint: Scopelint checks for unpinned variables in go programs [fast: true, auto-fix: false]
-stylecheck: Stylecheck is a replacement for golint [fast: false, auto-fix: false]
+stylecheck: Stylecheck is a replacement for golint [fast: true, auto-fix: false]
 unconvert: Remove unnecessary type conversions [fast: true, auto-fix: false]
-unparam: Reports unused function parameters [fast: false, auto-fix: false]
+unparam: Reports unused function parameters [fast: true, auto-fix: false]
 whitespace: Tool for detection of leading and trailing whitespace [fast: true, auto-fix: true]
 ```
 
@@ -462,7 +462,7 @@ golangci-lint help linters
 - [scopelint](https://github.com/kyoh86/scopelint) - Scopelint checks for unpinned variables in go programs
 - [gocritic](https://github.com/go-critic/go-critic) - The most opinionated Go source code linter
 - [gochecknoinits](https://github.com/leighmcculloch/gochecknoinits) - Checks that no init functions are present in Go code
-- [gochecknoglobals](https://github.com/leighmcculloch/gochecknoglobals) - Checks that no globals are present in Go code
+- [gochecknoglobals](https://github.com/leighmcculloch/gochecknoglobals) - Tool for detection of long functions
 - [godox](https://github.com/matoous/godox) - Tool for detection of FIXME, TODO and other comment keywords
 - [funlen](https://github.com/ultraware/funlen) - Tool for detection of long functions
 - [whitespace](https://github.com/ultraware/whitespace) - Tool for detection of leading and trailing whitespace
@@ -563,6 +563,7 @@ Global Flags:
       --mem-profile-path string   Path to memory profile output file
       --trace-path string         Path to trace output file
   -v, --verbose                   verbose output
+      --version                   Print version
 
 ```
 

cmd/golangci-lint/mod_version.go

@@ -10,7 +10,7 @@ import (
 //nolint:gochecknoinits
 func init() {
 	if info, available := debug.ReadBuildInfo(); available {
-		if date == "" && info.Main.Version != "(devel)" {
+		if date == "" {
 			version = info.Main.Version
 			commit = fmt.Sprintf("(unknown, mod sum: %q)", info.Main.Sum)
 			date = "(unknown)"

go.mod

@@ -13,9 +13,9 @@ require (
 	github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613
 	github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3
 	github.com/golangci/gocyclo v0.0.0-20180528134321-2becd97e67ee
-	github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98
+	github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a
 	github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc
-	github.com/golangci/lint-1 v0.0.0-20190420132249-ee948d087217
+	github.com/golangci/lint-1 v0.0.0-20190930103755-fad67e08aa89
 	github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca
 	github.com/golangci/misspell v0.0.0-20180809174111-950f5d19e770
 	github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21

go.sum

@@ -86,12 +86,12 @@ github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3 h1:pe9JHs3cHHDQgO
 github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3/go.mod h1:JXrF4TWy4tXYn62/9x8Wm/K/dm06p8tCKwFRDPZG/1o=
 github.com/golangci/gocyclo v0.0.0-20180528134321-2becd97e67ee h1:J2XAy40+7yz70uaOiMbNnluTg7gyQhtGqLQncQh+4J8=
 github.com/golangci/gocyclo v0.0.0-20180528134321-2becd97e67ee/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU=
-github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98 h1:0OkFarm1Zy2CjCiDKfK9XHgmc2wbDlRMD2hD8anAJHU=
-github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU=
+github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a h1:iR3fYXUjHCR97qWS8ch1y9zPNsgXThGwjKPrYfqMPks=
+github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU=
 github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc h1:gLLhTLMk2/SutryVJ6D4VZCU3CUqr8YloG7FPIBWFpI=
 github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc/go.mod h1:e5tpTHCfVze+7EpLEozzMB3eafxo2KT5veNg1k6byQU=
-github.com/golangci/lint-1 v0.0.0-20190420132249-ee948d087217 h1:En/tZdwhAn0JNwLuXzP3k2RVtMqMmOEK7Yu/g3tmtJE=
-github.com/golangci/lint-1 v0.0.0-20190420132249-ee948d087217/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg=
+github.com/golangci/lint-1 v0.0.0-20190930103755-fad67e08aa89 h1:664ewjIQUXDvinFMbAsoH2V2Yvaro/X8BoYpIMTWGXI=
+github.com/golangci/lint-1 v0.0.0-20190930103755-fad67e08aa89/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg=
 github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca h1:kNY3/svz5T29MYHubXix4aDDuE3RWHkPvopM/EDv/MA=
 github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca/go.mod h1:tvlJhZqDe4LMs4ZHD0oMUlt9G2LWuDGoisJTBzLMV9o=
 github.com/golangci/misspell v0.0.0-20180809174111-950f5d19e770 h1:EL/O5HGrF7Jaq0yNhBLucz9hTuRzj2LdwGBOaENgxIk=

pkg/commands/run.go

@@ -265,7 +265,7 @@ func fixSlicesFlags(fs *pflag.FlagSet) {
 	})
 }
 
-func (e *Executor) runAnalysis(ctx context.Context, args []string) (<-chan result.Issue, error) {
+func (e *Executor) runAnalysis(ctx context.Context, args []string) ([]result.Issue, error) {
 	e.cfg.Run.Args = args
 
 	enabledLinters, err := e.EnabledLintersSet.Get(true)
@@ -296,9 +296,9 @@ func (e *Executor) runAnalysis(ctx context.Context, args []string) (<-chan resul
 		return nil, err
 	}
 
-	issuesCh := runner.Run(ctx, enabledLinters, lintCtx)
+	issues := runner.Run(ctx, enabledLinters, lintCtx)
 	fixer := processors.NewFixer(e.cfg, e.log, e.fileCache)
-	return fixer.Process(issuesCh), nil
+	return fixer.Process(issues), nil
 }
 
 func (e *Executor) setOutputToDevNull() (savedStdout, savedStderr *os.File) {
@@ -313,24 +313,10 @@ func (e *Executor) setOutputToDevNull() (savedStdout, savedStderr *os.File) {
 	return
 }
 
-func (e *Executor) setExitCodeIfIssuesFound(issues <-chan result.Issue) <-chan result.Issue {
-	resCh := make(chan result.Issue, 1024)
-
-	go func() {
-		issuesFound := false
-		for i := range issues {
-			issuesFound = true
-			resCh <- i
-		}
-
-		if issuesFound {
-			e.exitCode = e.cfg.Run.ExitCodeIfIssuesFound
-		}
-
-		close(resCh)
-	}()
-
-	return resCh
+func (e *Executor) setExitCodeIfIssuesFound(issues []result.Issue) {
+	if len(issues) != 0 {
+		e.exitCode = e.cfg.Run.ExitCodeIfIssuesFound
+	}
 }
 
 func (e *Executor) runAndPrint(ctx context.Context, args []string) error {
@@ -357,7 +343,7 @@ func (e *Executor) runAndPrint(ctx context.Context, args []string) error {
 		return err
 	}
 
-	issues = e.setExitCodeIfIssuesFound(issues)
+	e.setExitCodeIfIssuesFound(issues)
 
 	if err = p.Print(ctx, issues); err != nil {
 		return fmt.Errorf("can't print %d issues: %s", len(issues), err)

pkg/golinters/bodyclose.go

@@ -17,5 +17,5 @@ func NewBodyclose() *goanalysis.Linter {
 		"checks whether HTTP response body is closed successfully",
 		analyzers,
 		nil,
-	)
+	).WithLoadMode(goanalysis.LoadModeTypesInfo)
 }

pkg/golinters/deadcode.go

@@ -1,42 +1,52 @@
 package golinters
 
 import (
-	"context"
 	"fmt"
+	"sync"
 
 	deadcodeAPI "github.com/golangci/go-misc/deadcode"
+	"golang.org/x/tools/go/analysis"
 
+	"github.com/golangci/golangci-lint/pkg/golinters/goanalysis"
 	"github.com/golangci/golangci-lint/pkg/lint/linter"
 	"github.com/golangci/golangci-lint/pkg/result"
 )
 
-type Deadcode struct{}
-
-func (Deadcode) Name() string {
-	return "deadcode"
-}
-
-func (Deadcode) Desc() string {
-	return "Finds unused code"
-}
-
-func (d Deadcode) Run(ctx context.Context, lintCtx *linter.Context) ([]result.Issue, error) {
-	issues, err := deadcodeAPI.Run(lintCtx.Program)
-	if err != nil {
-		return nil, err
-	}
-
-	if len(issues) == 0 {
-		return nil, nil
-	}
-
-	res := make([]result.Issue, 0, len(issues))
-	for _, i := range issues {
-		res = append(res, result.Issue{
-			Pos:        i.Pos,
-			Text:       fmt.Sprintf("%s is unused", formatCode(i.UnusedIdentName, lintCtx.Cfg)),
-			FromLinter: d.Name(),
-		})
+func NewDeadcode() *goanalysis.Linter {
+	const linterName = "deadcode"
+	var mu sync.Mutex
+	var resIssues []result.Issue
+
+	analyzer := &analysis.Analyzer{
+		Name: goanalysis.TheOnlyAnalyzerName,
+		Doc:  goanalysis.TheOnlyanalyzerDoc,
+		Run: func(pass *analysis.Pass) (interface{}, error) {
+			prog := goanalysis.MakeFakeLoaderProgram(pass)
+			issues, err := deadcodeAPI.Run(prog)
+			if err != nil {
+				return nil, err
+			}
+			res := make([]result.Issue, 0, len(issues))
+			for _, i := range issues {
+				res = append(res, result.Issue{
+					Pos:        i.Pos,
+					Text:       fmt.Sprintf("%s is unused", formatCode(i.UnusedIdentName, nil)),
+					FromLinter: linterName,
+				})
+			}
+			mu.Lock()
+			resIssues = append(resIssues, res...)
+			mu.Unlock()
+
+			return nil, nil
+		},
 	}
-	return res, nil
+	return goanalysis.NewLinter(
+		linterName,
+		"Finds unused code",
+		[]*analysis.Analyzer{analyzer},
+		nil,
+	).WithIssuesReporter(func(*linter.Context) []result.Issue {
+		return resIssues
+	}).WithLoadMode(goanalysis.LoadModeTypesInfo)
 }

pkg/golinters/depguard.go

@@ -1,22 +1,21 @@
 package golinters
 
 import (
-	"context"
 	"fmt"
 	"strings"
+	"sync"
+
+	"golang.org/x/tools/go/analysis"
+	"golang.org/x/tools/go/loader"
+
+	"github.com/golangci/golangci-lint/pkg/golinters/goanalysis"
 
 	depguardAPI "github.com/OpenPeeDeeP/depguard"
 
 	"github.com/golangci/golangci-lint/pkg/lint/linter"
 	"github.com/golangci/golangci-lint/pkg/result"
 )
 
-type Depguard struct{}
-
-func (Depguard) Name() string {
-	return "depguard"
-}
-
 func setDepguardListType(dg *depguardAPI.Depguard, lintCtx *linter.Context) error {
 	listType := lintCtx.Settings().Depguard.ListType
 	var found bool
@@ -49,42 +48,67 @@ func setupDepguardPackages(dg *depguardAPI.Depguard, lintCtx *linter.Context) {
 	}
 }
 
-func (Depguard) Desc() string {
-	return "Go linter that checks if package imports are in a list of acceptable packages"
-}
+func NewDepguard() *goanalysis.Linter {
+	const linterName = "depguard"
+	var mu sync.Mutex
+	var resIssues []result.Issue
 
-func (d Depguard) Run(ctx context.Context, lintCtx *linter.Context) ([]result.Issue, error) {
-	dg := &depguardAPI.Depguard{
-		Packages:      lintCtx.Settings().Depguard.Packages,
-		IncludeGoRoot: lintCtx.Settings().Depguard.IncludeGoRoot,
-	}
-	if err := setDepguardListType(dg, lintCtx); err != nil {
-		return nil, err
+	analyzer := &analysis.Analyzer{
+		Name: goanalysis.TheOnlyAnalyzerName,
+		Doc:  goanalysis.TheOnlyanalyzerDoc,
 	}
-	setupDepguardPackages(dg, lintCtx)
+	return goanalysis.NewLinter(
+		linterName,
+		"Go linter that checks if package imports are in a list of acceptable packages",
+		[]*analysis.Analyzer{analyzer},
+		nil,
+	).WithContextSetter(func(lintCtx *linter.Context) {
+		dgSettings := &lintCtx.Settings().Depguard
+		analyzer.Run = func(pass *analysis.Pass) (interface{}, error) {
+			prog := goanalysis.MakeFakeLoaderProgram(pass)
+			dg := &depguardAPI.Depguard{
+				Packages:      dgSettings.Packages,
+				IncludeGoRoot: dgSettings.IncludeGoRoot,
+			}
+			if err := setDepguardListType(dg, lintCtx); err != nil {
+				return nil, err
+			}
+			setupDepguardPackages(dg, lintCtx)
 
-	issues, err := dg.Run(lintCtx.LoaderConfig, lintCtx.Program)
-	if err != nil {
-		return nil, err
-	}
-	if len(issues) == 0 {
-		return nil, nil
-	}
-	msgSuffix := "is in the blacklist"
-	if dg.ListType == depguardAPI.LTWhitelist {
-		msgSuffix = "is not in the whitelist"
-	}
-	res := make([]result.Issue, 0, len(issues))
-	for _, i := range issues {
-		userSuppliedMsgSuffix := lintCtx.Settings().Depguard.PackagesWithErrorMessage[i.PackageName]
-		if userSuppliedMsgSuffix != "" {
-			userSuppliedMsgSuffix = ": " + userSuppliedMsgSuffix
+			loadConfig := &loader.Config{
+				Cwd:   "",  // fallbacked to os.Getcwd
+				Build: nil, // fallbacked to build.Default
+			}
+			issues, err := dg.Run(loadConfig, prog)
+			if err != nil {
+				return nil, err
+			}
+			if len(issues) == 0 {
+				return nil, nil
+			}
+			msgSuffix := "is in the blacklist"
+			if dg.ListType == depguardAPI.LTWhitelist {
+				msgSuffix = "is not in the whitelist"
+			}
+			res := make([]result.Issue, 0, len(issues))
+			for _, i := range issues {
+				userSuppliedMsgSuffix := dgSettings.PackagesWithErrorMessage[i.PackageName]
+				if userSuppliedMsgSuffix != "" {
+					userSuppliedMsgSuffix = ": " + userSuppliedMsgSuffix
+				}
+				res = append(res, result.Issue{
+					Pos:        i.Position,
+					Text:       fmt.Sprintf("%s %s%s", formatCode(i.PackageName, lintCtx.Cfg), msgSuffix, userSuppliedMsgSuffix),
+					FromLinter: linterName,
+				})
+			}
+			mu.Lock()
+			resIssues = append(resIssues, res...)
+			mu.Unlock()
+
+			return nil, nil
 		}
-		res = append(res, result.Issue{
-			Pos:        i.Position,
-			Text:       fmt.Sprintf("%s %s%s", formatCode(i.PackageName, lintCtx.Cfg), msgSuffix, userSuppliedMsgSuffix),
-			FromLinter: d.Name(),
-		})
-	}
-	return res, nil
+	}).WithIssuesReporter(func(*linter.Context) []result.Issue {
+		return resIssues
+	}).WithLoadMode(goanalysis.LoadModeTypesInfo)
 }