add AZT004: acceptance tests must be named TestAcc

CHANGELOG.md

@@ -1,5 +1,6 @@
 ## Unreleased
 
+- add rule `AZT004`: test functions that run an acceptance test but are not named `TestAcc`, so acceptance runs never pick them up; report only, no fix ([#67](https://github.com/katbyte/azproviderlint/pull/67))
 - add rule `AZT003`: test functions named `TestAcc` that never run an acceptance test, so are scheduled as one without touching Azure; report only, no fix ([#64](https://github.com/katbyte/azproviderlint/pull/64))
 
 ## v0.11.0 (2026-09-28)

README.md

@@ -105,6 +105,7 @@ Rules are named `AZ`, a category letter, and a number. The letters follow [tfpro
 | [AZT001](checks/AZT/AZT001_acceptance_test_external_package) | acceptance tests must use a _test package |
 | [AZT002](checks/AZT/AZT002_credentials_from_environment) | acceptance tests must not read credentials from the environment |
 | [AZT003](checks/AZT/AZT003_misnamed_acceptance_test) | TestAcc functions must run an acceptance test |
+| [AZT004](checks/AZT/AZT004_acceptance_test_not_named_test_acc) | acceptance tests must be named TestAcc |
 
 ### AZN - Naming Conventions
 

checks/AZT/AZT.go

@@ -7,11 +7,13 @@ import (
 	AZT001 "github.com/katbyte/azproviderlint/checks/AZT/AZT001_acceptance_test_external_package"
 	AZT002 "github.com/katbyte/azproviderlint/checks/AZT/AZT002_credentials_from_environment"
 	AZT003 "github.com/katbyte/azproviderlint/checks/AZT/AZT003_misnamed_acceptance_test"
+	AZT004 "github.com/katbyte/azproviderlint/checks/AZT/AZT004_acceptance_test_not_named_test_acc"
 )
 
 // Checks contains all AZT (acceptance testing) analyzers.
 var Checks = []*analysis.Analyzer{
 	AZT001.Analyzer,
 	AZT002.Analyzer,
 	AZT003.Analyzer,
+	AZT004.Analyzer,
 }

checks/AZT/AZT003_misnamed_acceptance_test/AZT003.go

@@ -4,122 +4,57 @@ package AZT003
 
 import (
 	"go/ast"
-	"go/constant"
 	"go/types"
-	"slices"
 	"strings"
 	"unicode"
 
 	"golang.org/x/tools/go/analysis"
 
-	"github.com/katbyte/azproviderlint/lib/astx"
+	"github.com/katbyte/azproviderlint/lib/acctest"
 )
 
 // Analyzer checks that every `TestAcc...` function reaches the acceptance test harness.
 // Acceptance runs select tests by that prefix, so a unit test carrying it is scheduled and
 // reported as an acceptance test although it never touches Azure.
 var Analyzer = &analysis.Analyzer{
-	Name: "AZT003",
-	Doc:  "check that test functions named TestAcc run an acceptance test",
-	URL:  "https://github.com/katbyte/azproviderlint/blob/main/checks/AZT/AZT003_misnamed_acceptance_test/README.md",
-	Run:  run,
-}
-
-// The Terraform test harnesses; a provider's own internal/acceptance package counts too.
-var harnessPackages = map[string]bool{
-	"github.com/hashicorp/terraform-plugin-testing/helper/resource": true,
-	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource":  true,
+	Name:     "AZT003",
+	Doc:      "check that test functions named TestAcc run an acceptance test",
+	URL:      "https://github.com/katbyte/azproviderlint/blob/main/checks/AZT/AZT003_misnamed_acceptance_test/README.md",
+	Requires: []*analysis.Analyzer{acctest.Analyzer},
+	Run:      run,
 }
 
 func run(pass *analysis.Pass) (any, error) {
-	decls := map[*types.Func]*ast.FuncDecl{}
-	var candidates []*types.Func
+	runs, ok := pass.ResultOf[acctest.Analyzer].(acctest.Funcs)
+	if !ok {
+		return nil, nil
+	}
+
 	for _, file := range pass.Files {
-		isTest := strings.HasSuffix(pass.Fset.Position(file.Pos()).Filename, "_test.go")
+		if !strings.HasSuffix(pass.Fset.Position(file.Pos()).Filename, "_test.go") {
+			continue
+		}
 		for _, decl := range file.Decls {
 			fd, ok := decl.(*ast.FuncDecl)
-			if !ok || fd.Body == nil {
+			// the bare prefix, as test runners match it: TestAccountName is picked up too
+			if !ok || fd.Body == nil || fd.Recv != nil || !strings.HasPrefix(fd.Name.Name, "TestAcc") {
 				continue
 			}
+
 			fn, ok := pass.TypesInfo.Defs[fd.Name].(*types.Func)
-			if !ok {
+			if !ok || runs[fn] != nil {
 				continue
 			}
-			decls[fn] = fd
-			// the bare prefix, as test runners match it: TestAccountName is picked up too
-			if isTest && fd.Recv == nil && strings.HasPrefix(fd.Name.Name, "TestAcc") {
-				candidates = append(candidates, fn)
-			}
-		}
-	}
-	if len(candidates) == 0 {
-		return nil, nil
-	}
-
-	// runs marks the functions that are acceptance tests on their own: they hand t to a
-	// harness or look at TF_ACC. mentions lists the same-package functions each one calls or
-	// passes around as a value (`t.Run("basic", testAccThing_basic)`).
-	runs := map[*types.Func]bool{}
-	mentions := map[*types.Func][]*types.Func{}
-	for fn, fd := range decls {
-		ast.Inspect(fd.Body, func(n ast.Node) bool {
-			switch n := n.(type) {
-			case *ast.CallExpr:
-				callee := astx.CalledFunc(pass, n)
-				if callee == nil || callee.Pkg() == nil {
-					break
-				}
-				if path := callee.Pkg().Path(); !harnessPackages[path] && !strings.HasSuffix(path, "/internal/acceptance") {
-					break
-				}
-				for _, arg := range n.Args {
-					ptr, ok := pass.TypesInfo.TypeOf(arg).(*types.Pointer)
-					if !ok {
-						continue
-					}
-					if named, ok := ptr.Elem().(*types.Named); ok && named.Obj().Name() == "T" && named.Obj().Pkg() != nil && named.Obj().Pkg().Path() == "testing" {
-						runs[fn] = true
-					}
-				}
-			case *ast.Ident:
-				if used, ok := pass.TypesInfo.Uses[n].(*types.Func); ok && used.Pkg() == pass.Pkg {
-					mentions[fn] = append(mentions[fn], used.Origin())
-				}
-			}
-			// by value, so resource.EnvTfAcc and local constants count as well as the literal
-			if e, ok := n.(ast.Expr); ok {
-				if v := pass.TypesInfo.Types[e].Value; v != nil && v.Kind() == constant.String && constant.StringVal(v) == "TF_ACC" {
-					runs[fn] = true
-				}
-			}
-			return true
-		})
-	}
 
-	// a function that mentions an acceptance test is one too
-	for changed := true; changed; {
-		changed = false
-		for fn, used := range mentions {
-			if !runs[fn] && slices.ContainsFunc(used, func(u *types.Func) bool { return runs[u] }) {
-				runs[fn] = true
-				changed = true
+			// Acc that starts a word (TestAccountName) cannot simply be dropped
+			rest := strings.TrimPrefix(fn.Name(), "TestAcc")
+			if rest != "" && unicode.IsLower(rune(rest[0])) {
+				pass.Reportf(fd.Name.Pos(), "%s does not run an acceptance test, rename it so it does not start with TestAcc and is not picked up as one", fn.Name())
+				continue
 			}
-		}
-	}
 
-	for _, fn := range candidates {
-		if runs[fn] {
-			continue
-		}
-		// Acc that starts a word (TestAccountName) cannot simply be dropped
-		rest := strings.TrimPrefix(fn.Name(), "TestAcc")
-		if rest != "" && unicode.IsLower(rune(rest[0])) {
-			pass.Reportf(decls[fn].Name.Pos(),
-				"%s does not run an acceptance test, rename it so it does not start with TestAcc and is not picked up as one", fn.Name())
-			continue
+			pass.Reportf(fd.Name.Pos(), "%s does not run an acceptance test, name it Test%s so it is not picked up as one", fn.Name(), rest)
 		}
-		pass.Reportf(decls[fn].Name.Pos(),
-			"%s does not run an acceptance test, name it Test%s so it is not picked up as one", fn.Name(), rest)
 	}
 
 	return nil, nil

checks/AZT/AZT003_misnamed_acceptance_test/testdata/src/github.com/hashicorp/terraform-plugin-testing/helper/resource/resource.go

@@ -19,3 +19,6 @@ type TestCase struct {
 func Test(t T, c TestCase) {}
 
 func ParallelTest(t T, c TestCase) {}
+
+// UnitTest runs the case without TF_ACC: a unit test on the harness, not an acceptance test.
+func UnitTest(t T, c TestCase) {}

checks/AZT/AZT003_misnamed_acceptance_test/testdata/src/github.com/hashicorp/terraform-provider-azurerm/internal/services/azt003/thing.go

@@ -1,6 +1,9 @@
 package azt003
 
-import "testing"
+import (
+	"os"
+	"testing"
+)
 
 func BuildName(prefix string) string {
 	return prefix + "-thing"
@@ -10,3 +13,9 @@ func BuildName(prefix string) string {
 func TestAccNotInATestFile(t *testing.T) {
 	_ = BuildName("a")
 }
+
+// Configured is production code that reads TF_ACC to pick a default; calling it does not make
+// a test an acceptance test
+func Configured() bool {
+	return os.Getenv("TF_ACC") != ""
+}

checks/AZT/AZT003_misnamed_acceptance_test/testdata/src/github.com/hashicorp/terraform-provider-azurerm/internal/services/azt003/thing_resource_test.go

@@ -156,6 +156,18 @@ func TestAccThing_randomName(t *testing.T) { // want `TestAccThing_randomName do
 	}
 }
 
+// Should be flagged: the harness's UnitTest runs without TF_ACC, so this is a unit test
+func TestAccThing_unitHarness(t *testing.T) { // want `TestAccThing_unitHarness does not run an acceptance test, name it TestThing_unitHarness so it is not picked up as one`
+	resource.UnitTest(t, resource.TestCase{Steps: []resource.TestStep{{Config: "unit"}}})
+}
+
+// Should be flagged: production code reading TF_ACC does not make this an acceptance test
+func TestAccThing_configured(t *testing.T) { // want `TestAccThing_configured does not run an acceptance test, name it TestThing_configured so it is not picked up as one`
+	if !azt003.Configured() {
+		t.Skip("not configured")
+	}
+}
+
 // Should be flagged: Acc starts a word here, but test runners still match the prefix
 func TestAccountName(t *testing.T) { // want `TestAccountName does not run an acceptance test, rename it so it does not start with TestAcc and is not picked up as one`
 	if azt003.BuildName("a") != "a-thing" {

checks/AZT/AZT004_acceptance_test_not_named_test_acc/AZT004.go

@@ -0,0 +1,66 @@
+// Package AZT004 defines an analyzer that reports test functions that run an acceptance test
+// but are not named TestAcc.
+package AZT004
+
+import (
+	"go/ast"
+	"go/types"
+	"regexp"
+	"strings"
+
+	"golang.org/x/tools/go/analysis"
+
+	"github.com/katbyte/azproviderlint/lib/acctest"
+)
+
+// Analyzer checks that every test function reaching the acceptance test harness is named
+// `TestAcc...`. Acceptance runs select tests by that prefix, so one named otherwise is never
+// run against Azure, however complete it is.
+var Analyzer = &analysis.Analyzer{
+	Name:     "AZT004",
+	Doc:      "check that test functions running an acceptance test are named TestAcc",
+	URL:      "https://github.com/katbyte/azproviderlint/blob/main/checks/AZT/AZT004_acceptance_test_not_named_test_acc/README.md",
+	Requires: []*analysis.Analyzer{acctest.Analyzer},
+	Run:      run,
+}
+
+// strayAcc matches Acc as a whole word inside a camel-case name: followed by an upper-case
+// letter, an underscore or the end, so Account and Access are not matched.
+var strayAcc = regexp.MustCompile(`Acc([A-Z_]|$)`)
+
+func run(pass *analysis.Pass) (any, error) {
+	runs, ok := pass.ResultOf[acctest.Analyzer].(acctest.Funcs)
+	if !ok {
+		return nil, nil
+	}
+
+	for _, file := range pass.Files {
+		if !strings.HasSuffix(pass.Fset.Position(file.Pos()).Filename, "_test.go") {
+			continue
+		}
+		for _, decl := range file.Decls {
+			fd, ok := decl.(*ast.FuncDecl)
+			// what go test runs: a top-level Test function; its signature is enforced by the compiler
+			if !ok || fd.Body == nil || fd.Recv != nil || !strings.HasPrefix(fd.Name.Name, "Test") || strings.HasPrefix(fd.Name.Name, "TestAcc") {
+				continue
+			}
+
+			fn, ok := pass.TypesInfo.Defs[fd.Name].(*types.Func)
+			if !ok || runs[fn] == nil {
+				continue
+			}
+
+			// Acc already sits somewhere in the name as its own word (TestWebAppAccActiveSlot_basic),
+			// so prefixing it would double up; Account and Access do not count
+			rest := strings.TrimPrefix(fn.Name(), "Test")
+			if strayAcc.MatchString(rest) {
+				pass.Reportf(fd.Name.Pos(), "%s runs an acceptance test, rename it to start with TestAcc so acceptance runs pick it up", fn.Name())
+				continue
+			}
+
+			pass.Reportf(fd.Name.Pos(), "%s runs an acceptance test, name it TestAcc%s so acceptance runs pick it up", fn.Name(), rest)
+		}
+	}
+
+	return nil, nil
+}

checks/AZT/AZT004_acceptance_test_not_named_test_acc/AZT004_test.go

@@ -0,0 +1,16 @@
+package AZT004
+
+import (
+	"path/filepath"
+	"runtime"
+	"testing"
+
+	"golang.org/x/tools/go/analysis/analysistest"
+)
+
+func TestAZT004(t *testing.T) {
+	t.Parallel()
+
+	_, filename, _, _ := runtime.Caller(0)
+	analysistest.Run(t, filepath.Join(filepath.Dir(filename), "testdata"), Analyzer, "github.com/hashicorp/terraform-provider-azurerm/internal/services/azt004")
+}

checks/AZT/AZT004_acceptance_test_not_named_test_acc/README.md

@@ -0,0 +1,61 @@
+# AZT004 - acceptance tests must be named TestAcc
+
+AZT004 reports test functions that run an acceptance test but are not named `TestAcc...`.
+
+Acceptance runs pick tests by that prefix. A test named `TestThing_basic` that drives the acceptance framework is skipped by every acceptance run, so the resource it covers is never actually tested against Azure, however complete the test looks.
+
+A function counts as an acceptance test when it does one of these, itself or through another function in the same package:
+
+- hands `t` to the provider's `internal/acceptance` package (`acceptance.BuildTestData(t, ...)`, `data.ResourceTest(t, ...)`, `acceptance.RunTestsInSequence(t, ...)`)
+- hands `t` to the Terraform test harness (`resource.Test`, `resource.ParallelTest`)
+- looks at `TF_ACC`
+
+Only functions go test runs are checked: a top-level `func TestXxx(t *testing.T)` in a `_test.go` file. Lowercase helpers and methods are left alone, since no runner selects them by name.
+
+## Flagged Code
+
+```go
+func TestThing_basic(t *testing.T) {
+	data := acceptance.BuildTestData(t, "azurerm_thing", "test")
+	r := ThingResource{}
+
+	data.ResourceTest(t, r, []acceptance.TestStep{
+		{
+			Config: r.basic(data),
+		},
+		data.ImportStep(),
+	})
+}
+```
+
+## Passing Code
+
+```go
+func TestAccThing_basic(t *testing.T) {
+	data := acceptance.BuildTestData(t, "azurerm_thing", "test")
+	r := ThingResource{}
+
+	data.ResourceTest(t, r, []acceptance.TestStep{
+		{
+			Config: r.basic(data),
+		},
+		data.ImportStep(),
+	})
+}
+```
+
+## No fix
+
+AZT004 only reports. Rename the function so it starts with `TestAcc`. When `Acc` already sits elsewhere in the name (`TestWebAppAccActiveSlot_basic`), move it to the front rather than adding a second one.
+
+This is the mirror of [AZT003](../AZT003_misnamed_acceptance_test), which reports the opposite: a `TestAcc` name on a test that never runs an acceptance test.
+
+## Ignoring Reports
+
+Put `//azignore:AZT004 - <reason>` at the end of the line, or on the line above it. The reason is required.
+
+```go
+func TestThing_basic(t *testing.T) { //azignore:AZT004 - <reason>
+```
+
+Under golangci-lint, `//nolint:azproviderlint` in the same place also works, but it silences every azproviderlint check on that line.

checks/AZT/AZT004_acceptance_test_not_named_test_acc/testdata/src/github.com/hashicorp/terraform-plugin-testing/helper/resource/resource.go

@@ -0,0 +1,24 @@
+// Package resource is a minimal stub of the terraform-plugin-testing harness.
+package resource
+
+const EnvTfAcc = "TF_ACC"
+
+// T mirrors the harness's own testing interface, which *testing.T satisfies.
+type T interface {
+	Fatal(args ...any)
+}
+
+type TestStep struct {
+	Config string
+}
+
+type TestCase struct {
+	Steps []TestStep
+}
+
+func Test(t T, c TestCase) {}
+
+func ParallelTest(t T, c TestCase) {}
+
+// UnitTest runs the case without TF_ACC: a unit test on the harness, not an acceptance test.
+func UnitTest(t T, c TestCase) {}