feat: Initial setup for conformance test suite

Makefile

@@ -121,7 +121,7 @@ vet: ## Run go vet against code.
 
 .PHONY: test
 test: manifests generate fmt vet envtest image-build ## Run tests.
-	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e) -race -coverprofile cover.out
+	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e | grep -v /conformance) -race -coverprofile cover.out
 
 .PHONY: test-integration
 test-integration: ## Run tests.

conformance/conformance.go

@@ -0,0 +1,233 @@
+/*
+Copyright 2025 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package conformance contains the core setup and execution logic
+// for the Gateway API Inference Extension conformance test suite.
+package conformance
+
+import (
+	"fmt"
+	"io/fs"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	clientset "k8s.io/client-go/kubernetes"
+
+	// Import runtime package for scheme creation
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/config"
+	"sigs.k8s.io/yaml"
+
+	// Import necessary types and utilities from the core Gateway API conformance suite.
+	// Assumes sigs.k8s.io/gateway-api is a dependency in the go.mod.
+	gatewayv1 "sigs.k8s.io/gateway-api/apis/v1"            // Import core Gateway API types
+	confapis "sigs.k8s.io/gateway-api/conformance/apis/v1" // Report struct definition
+	confconfig "sigs.k8s.io/gateway-api/conformance/utils/config"
+	confflags "sigs.k8s.io/gateway-api/conformance/utils/flags"
+	confsuite "sigs.k8s.io/gateway-api/conformance/utils/suite"
+	"sigs.k8s.io/gateway-api/pkg/features" // Using core features definitions if applicable
+
+	// Import the test definitions package to access the ConformanceTests slice
+	"sigs.k8s.io/gateway-api-inference-extension/conformance/tests"
+
+	// == Import test packages using blank identifier ==
+	// This triggers the init() functions in these packages, which register the tests
+	// by appending them to the tests.ConformanceTests slice.
+	_ "sigs.k8s.io/gateway-api-inference-extension/conformance/tests/basic"
+	// TODO: Add blank imports for other test categories as they are created.
+	// _ "sigs.k8s.io/gateway-api-inference-extension/conformance/tests/model_routing"
+
+	// == Import the Inference Extension API types ==
+	inferencev1alpha2 "sigs.k8s.io/gateway-api-inference-extension/api/v1alpha2"
+)
+
+// ConformanceProfileName defines the name for the Inference Extension conformance profile.
+const InferenceConformanceProfileName confsuite.ConformanceProfileName = "InferenceGateway"
+
+var InferenceCoreFeatures = sets.New[features.FeatureName]() // Placeholder - Populate with actual features
+
+// Define the Inference Gateway conformance profile.
+// As per the design doc, initially, there are no "extended" features;
+// all tested features are considered mandatory for this profile.
+var InferenceGatewayProfile = confsuite.ConformanceProfile{
+	Name:         InferenceConformanceProfileName,
+	CoreFeatures: InferenceCoreFeatures,
+}
+
+// DefaultOptions parses command line flags and sets up the suite options.
+// Adapted from the core Gateway API conformance suite.
+func DefaultOptions(t *testing.T) confsuite.ConformanceOptions {
+	t.Helper()
+
+	cfg, err := config.GetConfig()
+	require.NoError(t, err, "error loading Kubernetes config")
+
+	// Initialize client options. The scheme must include Gateway API types
+	// and the Inference Extension types.
+	clientOptions := client.Options{}
+	scheme := clientOptions.Scheme
+	if scheme == nil {
+		// If default options don't provide a scheme, create one using runtime.NewScheme().
+		scheme = runtime.NewScheme()
+		clientOptions.Scheme = scheme
+	}
+
+	// Register necessary API Types
+	require.NoError(t, gatewayv1.Install(scheme)) // Add core Gateway API types
+	// Add the Inference Extension API types to the scheme using the correct import alias
+	require.NoError(t, inferencev1alpha2.Install(scheme))
+	require.NoError(t, apiextensionsv1.AddToScheme(scheme)) // Needed for CRD checks
+
+	// Create the Kubernetes clients
+	c, err := client.New(cfg, clientOptions)
+	require.NoError(t, err, "error initializing Kubernetes client")
+	cs, err := clientset.NewForConfig(cfg)
+	require.NoError(t, err, "error initializing Kubernetes clientset")
+
+	// Parse flags using the core Gateway API flag definitions
+	// supportedFeatures := confsuite.ParseSupportedFeatures(*confflags.SupportedFeatures)
+	// We derive this from the profile
+	exemptFeatures := confsuite.ParseSupportedFeatures(*confflags.ExemptFeatures)
+	skipTests := confsuite.ParseSkipTests(*confflags.SkipTests)
+	namespaceLabels := confsuite.ParseKeyValuePairs(*confflags.NamespaceLabels)
+	namespaceAnnotations := confsuite.ParseKeyValuePairs(*confflags.NamespaceAnnotations)
+	// For this suite, we only run the InferenceGateway profile
+	conformanceProfiles := sets.New(InferenceConformanceProfileName)
+
+	// Implementation details from flags
+	implementation := confsuite.ParseImplementation(
+		*confflags.ImplementationOrganization,
+		*confflags.ImplementationProject,
+		*confflags.ImplementationURL,
+		*confflags.ImplementationVersion,
+		*confflags.ImplementationContact,
+	)
+
+	// --- Inference Extension Specific Report Fields ---
+	// TODO: Determine the source for these values.
+	inferenceExtensionChannel := "experimental"
+	inferenceExtensionVersion := "v1alpha2"
+	_ = inferenceExtensionChannel // Avoid unused variable error until implemented
+	_ = inferenceExtensionVersion // Avoid unused variable error until implemented
+
+	// Create ConformanceOptions
+	opts := confsuite.ConformanceOptions{
+		Client:               c,
+		Clientset:            cs,
+		RestConfig:           cfg,
+		GatewayClassName:     *confflags.GatewayClassName,
+		Debug:                *confflags.ShowDebug,
+		CleanupBaseResources: *confflags.CleanupBaseResources,
+		SupportedFeatures:    sets.New[features.FeatureName](), // Initialize empty, will be populated below
+		TimeoutConfig:        confconfig.DefaultTimeoutConfig(),
+		SkipTests:            skipTests,
+		ExemptFeatures:       exemptFeatures,
+		RunTest:              *confflags.RunTest,
+		Mode:                 *confflags.Mode,
+		NamespaceLabels:      namespaceLabels,
+		NamespaceAnnotations: namespaceAnnotations,
+		Implementation:       implementation,
+		ConformanceProfiles:  conformanceProfiles,
+		ManifestFS:           []fs.FS{&Manifests}, // Assumes embed.go defines `Manifests`
+		ReportOutputPath:     *confflags.ReportOutput,
+		SkipProvisionalTests: *confflags.SkipProvisionalTests,
+		// TODO: Add the inference extension specific fields to ConformanceOptions struct if needed,
+		// or handle them during report generation.
+		// GatewayAPIInferenceExtensionChannel: inferenceExtensionChannel,
+		// GatewayAPIInferenceExtensionVersion: inferenceExtensionVersion,
+	}
+
+	// Populate SupportedFeatures based on the InferenceGateway profile.
+	// Since all features are mandatory for this profile, add all defined core features.
+	if opts.ConformanceProfiles.Has(InferenceConformanceProfileName) {
+		for feature := range InferenceGatewayProfile.CoreFeatures {
+			opts.SupportedFeatures.Insert(feature)
+		}
+	}
+
+	// Remove any features explicitly exempted via flags.
+	for feature := range opts.ExemptFeatures {
+		opts.SupportedFeatures.Delete(feature)
+	}
+
+	return opts
+}
+
+// RunConformance runs the Inference Extension conformance tests using default options.
+func RunConformance(t *testing.T) {
+	RunConformanceWithOptions(t, DefaultOptions(t))
+}
+
+// RunConformanceWithOptions runs the Inference Extension conformance tests with specific options.
+func RunConformanceWithOptions(t *testing.T, opts confsuite.ConformanceOptions) {
+	t.Logf("Running Inference Extension conformance tests with GatewayClass %s", opts.GatewayClassName)
+
+	// Register the InferenceGateway profile with the suite runner.
+	confsuite.RegisterConformanceProfile(InferenceGatewayProfile)
+
+	// Initialize the test suite.
+	cSuite, err := confsuite.NewConformanceTestSuite(opts)
+	require.NoError(t, err, "error initializing conformance suite")
+
+	t.Log("Setting up Inference Extension conformance tests")
+	// Setup requires the list of tests, which is populated by the init() functions
+	// triggered by the blank imports at the top of this file.
+	cSuite.Setup(t, tests.ConformanceTests)
+
+	t.Log("Running Inference Extension conformance tests")
+	// Run the tests.
+	err = cSuite.Run(t, tests.ConformanceTests)
+	require.NoError(t, err, "error running conformance tests")
+
+	// Generate and write the report if requested.
+	if opts.ReportOutputPath != "" {
+		t.Log("Generating Inference Extension conformance report")
+		report, err := cSuite.Report() // Use the existing report generation logic.
+		require.NoError(t, err, "error generating conformance report")
+
+		// TODO: Modify the report struct here if channel, version need to be modified.
+		// Example (requires adding fields to confapis.ConformanceReport):
+		// report.GatewayAPIInferenceExtensionChannel = opts.GatewayAPIInferenceExtensionChannel
+		// report.GatewayAPIInferenceExtensionVersion = opts.GatewayAPIInferenceExtensionVersion
+
+		err = writeReport(t.Logf, *report, opts.ReportOutputPath)
+		require.NoError(t, err, "error writing conformance report")
+	}
+}
+
+// writeReport writes the generated conformance report to the specified output file or logs it.
+// Adapted from the core Gateway API suite.
+func writeReport(logf func(string, ...any), report confapis.ConformanceReport, output string) error {
+	rawReport, err := yaml.Marshal(report)
+	if err != nil {
+		return fmt.Errorf("error marshaling report: %w", err)
+	}
+
+	if output != "" {
+		if err = os.WriteFile(output, rawReport, 0o600); err != nil {
+			return fmt.Errorf("error writing report file %s: %w", output, err)
+		}
+		logf("Conformance report written to %s", output)
+	} else {
+		// Log the report YAML to stdout if no output file is specified.
+		logf("Conformance report:\n%s", string(rawReport))
+	}
+	return nil
+}

conformance/conformance_test.go

@@ -0,0 +1,29 @@
+/*
+Copyright 2025 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package conformance
+
+import (
+	"testing"
+)
+
+// TestConformance is the top-level function that runs the conformance tests.
+// It calls the RunConformance function which sets up the suite and executes
+// the registered tests.
+func TestConformance(t *testing.T) {
+	// RunConformance is defined in conformance.go
+	RunConformance(t)
+}

conformance/embed.go

@@ -0,0 +1,25 @@
+/*
+Copyright 2025 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package conformance
+
+import "embed"
+
+// Manifests embeds the contents of the conformance/resources directory making
+// the YAML files within them available to the test suite at runtime.
+//
+//go:embed resources/* tests/*
+var Manifests embed.FS

conformance/reports/README.md

@@ -0,0 +1,88 @@
+# Conformance Reports for Gateway API Inference Extension
+
+This directory stores conformance reports submitted by various implementations of the Gateway API Inference Extension. This structure closely follows the [kubernetes-sigs/gateway-api/conformance/reports](https://git.1-hub.cnkubernetes-sigs/gateway-api/blob/main/conformance/reports/README).
+
+## How this folder is structured
+
+This folder stores conformance reports organized by the version of the Gateway API Inference Extension specification they were tested against:
+
+|-- conformance/reports
+|   |-- v0.3.0  # Example version
+|   |   |-- my-inference-gateway
+|   |   |   |-- README.md
+|   |   |   |-- experimental-v1.2.3-default-report.yaml # Example report file
+|   |   |-- another-implementation
+|   |   |   |-- README.md
+|   |   |   |-- ...
+|   |-- v0.4.0 # Future version
+|   |   |-- ...
+
+## Implementation Submissions
+
+Each implementation conformant with a specific version of the Gateway API Inference Extension should have its own folder within the corresponding version directory (e.g., `/conformance/reports/v0.3.0/my-implementation/`).
+
+The implementation is the owner of its folder and is responsible for:
+
+1.  Uploading one or more conformance reports (YAML files).
+2.  Maintaining a mandatory `README.md` file within their folder, structured as follows:
+
+    # My Inference Gateway Implementation
+
+    General information about the My/Implementation project.
+
+    ## Table of Contents
+
+    | Extension Channel | Extension Version Tested | Implementation Version | Mode    | Report                                            |
+    |-------------------|--------------------------|------------------------|---------|---------------------------------------------------|
+    | experimental      | v0.3.0                   | v1.2.3                 | default | [v1.2.3 report](./experimental-v1.2.3-default-report.yaml) |
+    | ...               | ...                      | ...                    | ...     | ...                                               |
+
+    ## Reproduce
+
+    Instructions on how to reproduce the claimed report(s).
+
+### Table of Contents (within Implementation README)
+
+The table of contents within an implementation's `README.md` should contain one row for each submitted report and include the following columns:
+
+* **Extension Channel**: The channel of the Gateway API Inference Extension (e.g., `standard` or `experimental`). Must correspond to the `gatewayAPIInferenceExtensionChannel` field in the report.
+* **Extension Version Tested**: The version of the Gateway API Inference Extension specification tested against (e.g., `v0.3.0`). Must correspond to the `gatewayAPIInferenceExtensionVersion` field in the report.
+* **Implementation Version**: A link to the GitHub/website page for the specific release/commit of the implementation tested. The version value MUST correspond to the `implementation.version` field in the report.
+* **Mode**: The operating mode of the implementation used for the test run (default is `default`). Must correspond to the `mode` field in the report. If a mode other than `default` is used, the "Reproduce" section must explain how to configure it.
+* **Report**: A link to the corresponding report YAML file. Reports MUST be named according to the pattern: `<Extension Channel>-<Implementation Version>-<Mode>-report.yaml`.
+
+### Reproduce Section (within Implementation README)
+
+This section MUST exist and contain the manual or automatic steps required to reproduce the results claimed by the uploaded conformance reports for that specific implementation. If reproduction steps differ significantly between implementation versions, use sub-sections.
+
+## Report Files
+
+Conformance reports MUST be uploaded exactly as generated by the official Gateway API Inference Extension conformance test suite, without any modifications. The "Reproduce" section allows for verification of the submitted report against a fresh run.
+
+### Report Rules
+
+To be accepted, submitted conformance reports must comply with the following rules:
+
+1.  **Implementation Details:** All fields within the `implementation` block must have meaningful values:
+    * `organization`: The entity maintaining the implementation (company, open source org, individual).
+    * `project`: The name of the implementation project, unique within the organization.
+    * `url`: A valid URL for the project (e.g., GitHub repository, product page).
+    * `version`: A specific, reproducible snapshot of the implementation (e.g., tag, commit hash, release version). Branch names are not acceptable.
+    * `contact`: A list of contact points (GitHub handles like `@maintainer`, team handles like `@org/team`, email addresses, or support URLs like an issue tracker).
+2.  **Inference Extension Versioning:** The report MUST include:
+    * `gatewayAPIInferenceExtensionVersion`: The specific version of the Gateway API Inference Extension specification tested against (e.g., `v0.3.0`).
+    * `gatewayAPIInferenceExtensionChannel`: The channel (`standard` or `experimental`) corresponding to the tested version.
+3.  **Mode:** The `mode` field indicates the implementation's operating mode during the test run.
+4.  **Test Profile & Result:**
+    * The report MUST contain exactly one profile result under the `profiles` list.
+    * This profile MUST have `name: InferenceGateway`.
+    * The profile's `result` field MUST be `success`. A `success` result indicates that **all** tests defined within the Gateway API Inference Extension conformance suite for the specified version passed. (Note: Unlike core Gateway API, there are currently no optional "extended" features within the Inference Extension conformance profile itself; all included tests are considered mandatory for conformance).
+
+## Submission Process
+
+Conformance reports demonstrating a `success` result for the `InferenceGateway` profile should be submitted via Pull Request directly to this repository (`kubernetes-sigs/gateway-api-inference-extension`).
+
+1.  Create a new folder under `/conformance/reports/<extension-version>/` named after your implementation (e.g., `/conformance/reports/v0.3.0/my-implementation/`).
+2.  Add your implementation's `README.md` to this folder, following the structure described above.
+3.  Add your generated conformance report YAML file(s) to this folder, ensuring they follow the naming convention `<Extension Channel>-<Implementation Version>-<Mode>-report.yaml`.
+4.  Submit the Pull Request.