From 78284df81b5728d357f57fd5e29113170747fbbe Mon Sep 17 00:00:00 2001 From: Sina Chavoshi Date: Mon, 21 Apr 2025 19:02:33 +0000 Subject: [PATCH 01/15] feat: Initial setup for conformance test suite --- conformance/conformance.go | 230 ++++++++++++++++++ conformance/conformance_test.go | 29 +++ conformance/embed.go | 25 ++ conformance/reports/README.md | 88 +++++++ .../resources/manifests/manifests.yaml | 104 ++++++++ .../tests/basic/inferencepool_accepted.go | 76 ++++++ .../tests/basic/inferencepool_accepted.yaml | 26 ++ conformance/tests/main.go | 35 +++ conformance/utils/assertions.go | 25 ++ conformance/utils/kubernetes/helpers.go | 46 ++++ conformance/utils/traffic/traffic.go | 22 ++ go.mod | 13 +- go.sum | 37 ++- 13 files changed, 729 insertions(+), 27 deletions(-) create mode 100644 conformance/conformance.go create mode 100644 conformance/conformance_test.go create mode 100644 conformance/embed.go create mode 100644 conformance/reports/README.md create mode 100644 conformance/resources/manifests/manifests.yaml create mode 100644 conformance/tests/basic/inferencepool_accepted.go create mode 100644 conformance/tests/basic/inferencepool_accepted.yaml create mode 100644 conformance/tests/main.go create mode 100644 conformance/utils/assertions.go create mode 100644 conformance/utils/kubernetes/helpers.go create mode 100644 conformance/utils/traffic/traffic.go diff --git a/conformance/conformance.go b/conformance/conformance.go new file mode 100644 index 00000000..e1583520 --- /dev/null +++ b/conformance/conformance.go @@ -0,0 +1,230 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package conformance contains the core setup and execution logic +// for the Gateway API Inference Extension conformance test suite. +package conformance + +import ( + "fmt" + "io/fs" + "os" + "testing" + + "github.com/stretchr/testify/require" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + clientset "k8s.io/client-go/kubernetes" + + // Import runtime package for scheme creation + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/sets" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/config" + "sigs.k8s.io/yaml" + + // Import necessary types and utilities from the core Gateway API conformance suite. + // Assumes sigs.k8s.io/gateway-api is a dependency in the go.mod. + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" // Import core Gateway API types + confapis "sigs.k8s.io/gateway-api/conformance/apis/v1" // Report struct definition + confconfig "sigs.k8s.io/gateway-api/conformance/utils/config" + confflags "sigs.k8s.io/gateway-api/conformance/utils/flags" + confsuite "sigs.k8s.io/gateway-api/conformance/utils/suite" + "sigs.k8s.io/gateway-api/pkg/features" // Using core features definitions if applicable + + // Import the test definitions package to access the ConformanceTests slice + "sigs.k8s.io/gateway-api-inference-extension/conformance/tests" + + // == Import test packages using blank identifier == + // This triggers the init() functions in these packages, which register the tests + // by appending them to the tests.ConformanceTests slice. + _ "sigs.k8s.io/gateway-api-inference-extension/conformance/tests/basic" + // TODO: Add blank imports for other test categories as they are created. + // _ "sigs.k8s.io/gateway-api-inference-extension/conformance/tests/model_routing" +) + +// ConformanceProfileName defines the name for the Inference Extension conformance profile. +const InferenceConformanceProfileName confsuite.ConformanceProfileName = "InferenceGateway" + +var InferenceCoreFeatures = sets.New[features.FeatureName]() // Placeholder - Populate with actual features + +// Define the Inference Gateway conformance profile. +// As per the design doc, initially, there are no "extended" features; +// all tested features are considered mandatory for this profile. +var InferenceGatewayProfile = confsuite.ConformanceProfile{ + Name: InferenceConformanceProfileName, + CoreFeatures: InferenceCoreFeatures, +} + +// DefaultOptions parses command line flags and sets up the suite options. +// Adapted from the core Gateway API conformance suite. +func DefaultOptions(t *testing.T) confsuite.ConformanceOptions { + t.Helper() + + cfg, err := config.GetConfig() + require.NoError(t, err, "error loading Kubernetes config") + + // Initialize client options. The scheme must include Gateway API types + // and the Inference Extension types. + clientOptions := client.Options{} + scheme := clientOptions.Scheme + if scheme == nil { + // If default options don't provide a scheme, create one using runtime.NewScheme(). + scheme = runtime.NewScheme() + clientOptions.Scheme = scheme + } + + // Register necessary API Types + require.NoError(t, gatewayv1.AddToScheme(scheme)) // Add core Gateway API types + // TODO: Add the Inference Extension API types to the scheme + // Example: require.NoError(t, inferencev1alpha1.AddToScheme(scheme)) + require.NoError(t, apiextensionsv1.AddToScheme(scheme)) // Needed for CRD checks + + // Create the Kubernetes clients + c, err := client.New(cfg, clientOptions) + require.NoError(t, err, "error initializing Kubernetes client") + cs, err := clientset.NewForConfig(cfg) + require.NoError(t, err, "error initializing Kubernetes clientset") + + // Parse flags using the core Gateway API flag definitions + // supportedFeatures := confsuite.ParseSupportedFeatures(*confflags.SupportedFeatures) + // We derive this from the profile + exemptFeatures := confsuite.ParseSupportedFeatures(*confflags.ExemptFeatures) + skipTests := confsuite.ParseSkipTests(*confflags.SkipTests) + namespaceLabels := confsuite.ParseKeyValuePairs(*confflags.NamespaceLabels) + namespaceAnnotations := confsuite.ParseKeyValuePairs(*confflags.NamespaceAnnotations) + // For this suite, we only run the InferenceGateway profile + conformanceProfiles := sets.New(InferenceConformanceProfileName) + + // Implementation details from flags + implementation := confsuite.ParseImplementation( + *confflags.ImplementationOrganization, + *confflags.ImplementationProject, + *confflags.ImplementationURL, + *confflags.ImplementationVersion, + *confflags.ImplementationContact, + ) + + // --- Inference Extension Specific Report Fields --- + // TODO: Determine the source for these values. + inferenceExtensionChannel := "experimental" + inferenceExtensionVersion := "v0.3.0" + _ = inferenceExtensionChannel // Avoid unused variable error until implemented + _ = inferenceExtensionVersion // Avoid unused variable error until implemented + + // Create ConformanceOptions + opts := confsuite.ConformanceOptions{ + Client: c, + Clientset: cs, + RestConfig: cfg, + GatewayClassName: *confflags.GatewayClassName, + Debug: *confflags.ShowDebug, + CleanupBaseResources: *confflags.CleanupBaseResources, + SupportedFeatures: sets.New[features.FeatureName](), // Initialize empty, will be populated below + TimeoutConfig: confconfig.DefaultTimeoutConfig(), + SkipTests: skipTests, + ExemptFeatures: exemptFeatures, + RunTest: *confflags.RunTest, + Mode: *confflags.Mode, + NamespaceLabels: namespaceLabels, + NamespaceAnnotations: namespaceAnnotations, + Implementation: implementation, + ConformanceProfiles: conformanceProfiles, + ManifestFS: []fs.FS{&Manifests}, // Assumes embed.go defines `Manifests` + ReportOutputPath: *confflags.ReportOutput, + SkipProvisionalTests: *confflags.SkipProvisionalTests, + // TODO: Add the inference extension specific fields to ConformanceOptions struct if needed, + // or handle them during report generation. + // GatewayAPIInferenceExtensionChannel: inferenceExtensionChannel, + // GatewayAPIInferenceExtensionVersion: inferenceExtensionVersion, + } + + // Populate SupportedFeatures based on the InferenceGateway profile. + // Since all features are mandatory for this profile, add all defined core features. + if opts.ConformanceProfiles.Has(InferenceConformanceProfileName) { + for feature := range InferenceGatewayProfile.CoreFeatures { + opts.SupportedFeatures.Insert(feature) + } + } + + // Remove any features explicitly exempted via flags. + for feature := range opts.ExemptFeatures { + opts.SupportedFeatures.Delete(feature) + } + + return opts +} + +// RunConformance runs the Inference Extension conformance tests using default options. +func RunConformance(t *testing.T) { + RunConformanceWithOptions(t, DefaultOptions(t)) +} + +// RunConformanceWithOptions runs the Inference Extension conformance tests with specific options. +func RunConformanceWithOptions(t *testing.T, opts confsuite.ConformanceOptions) { + t.Logf("Running Inference Extension conformance tests with GatewayClass %s", opts.GatewayClassName) + + // Register the InferenceGateway profile with the suite runner. + confsuite.RegisterConformanceProfile(InferenceGatewayProfile) + + // Initialize the test suite. + cSuite, err := confsuite.NewConformanceTestSuite(opts) + require.NoError(t, err, "error initializing conformance suite") + + t.Log("Setting up Inference Extension conformance tests") + // Setup requires the list of tests, which is populated by the init() functions + // triggered by the blank imports at the top of this file. + cSuite.Setup(t, tests.ConformanceTests) + + t.Log("Running Inference Extension conformance tests") + // Run the tests. + err = cSuite.Run(t, tests.ConformanceTests) + require.NoError(t, err, "error running conformance tests") + + // Generate and write the report if requested. + if opts.ReportOutputPath != "" { + t.Log("Generating Inference Extension conformance report") + report, err := cSuite.Report() // Use the existing report generation logic. + require.NoError(t, err, "error generating conformance report") + + // TODO: Modify the report struct here if channel, version need to be modified. + // Example (requires adding fields to confapis.ConformanceReport): + // report.GatewayAPIInferenceExtensionChannel = opts.GatewayAPIInferenceExtensionChannel + // report.GatewayAPIInferenceExtensionVersion = opts.GatewayAPIInferenceExtensionVersion + + err = writeReport(t.Logf, *report, opts.ReportOutputPath) + require.NoError(t, err, "error writing conformance report") + } +} + +// writeReport writes the generated conformance report to the specified output file or logs it. +// Adapted from the core Gateway API suite. +func writeReport(logf func(string, ...any), report confapis.ConformanceReport, output string) error { + rawReport, err := yaml.Marshal(report) + if err != nil { + return fmt.Errorf("error marshaling report: %w", err) + } + + if output != "" { + if err = os.WriteFile(output, rawReport, 0o600); err != nil { + return fmt.Errorf("error writing report file %s: %w", output, err) + } + logf("Conformance report written to %s", output) + } else { + // Log the report YAML to stdout if no output file is specified. + logf("Conformance report:\n%s", string(rawReport)) + } + return nil +} diff --git a/conformance/conformance_test.go b/conformance/conformance_test.go new file mode 100644 index 00000000..de82d5ec --- /dev/null +++ b/conformance/conformance_test.go @@ -0,0 +1,29 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package conformance + +import ( + "testing" +) + +// TestConformance is the top-level function that runs the conformance tests. +// It calls the RunConformance function which sets up the suite and executes +// the registered tests. +func TestConformance(t *testing.T) { + // RunConformance is defined in conformance.go + RunConformance(t) +} diff --git a/conformance/embed.go b/conformance/embed.go new file mode 100644 index 00000000..f7fa64c9 --- /dev/null +++ b/conformance/embed.go @@ -0,0 +1,25 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package conformance + +import "embed" + +// Manifests embeds the contents of the conformance/resources directory making +// the YAML files within them available to the test suite at runtime. +// +//go:embed resources/* tests/* +var Manifests embed.FS diff --git a/conformance/reports/README.md b/conformance/reports/README.md new file mode 100644 index 00000000..a16c8014 --- /dev/null +++ b/conformance/reports/README.md @@ -0,0 +1,88 @@ +# Conformance Reports for Gateway API Inference Extension + +This directory stores conformance reports submitted by various implementations of the Gateway API Inference Extension. This structure closely follows the [kubernetes-sigs/gateway-api/conformance/reports](https://github.com/kubernetes-sigs/gateway-api/blob/main/conformance/reports/README). + +## How this folder is structured + +This folder stores conformance reports organized by the version of the Gateway API Inference Extension specification they were tested against: + +|-- conformance/reports +| |-- v0.3.0 # Example version +| | |-- my-inference-gateway +| | | |-- README.md +| | | |-- experimental-v1.2.3-default-report.yaml # Example report file +| | |-- another-implementation +| | | |-- README.md +| | | |-- ... +| |-- v0.4.0 # Future version +| | |-- ... + +## Implementation Submissions + +Each implementation conformant with a specific version of the Gateway API Inference Extension should have its own folder within the corresponding version directory (e.g., `/conformance/reports/v0.3.0/my-implementation/`). + +The implementation is the owner of its folder and is responsible for: + +1. Uploading one or more conformance reports (YAML files). +2. Maintaining a mandatory `README.md` file within their folder, structured as follows: + + # My Inference Gateway Implementation + + General information about the My/Implementation project. + + ## Table of Contents + + | Extension Channel | Extension Version Tested | Implementation Version | Mode | Report | + |-------------------|--------------------------|------------------------|---------|---------------------------------------------------| + | experimental | v0.3.0 | v1.2.3 | default | [v1.2.3 report](./experimental-v1.2.3-default-report.yaml) | + | ... | ... | ... | ... | ... | + + ## Reproduce + + Instructions on how to reproduce the claimed report(s). + +### Table of Contents (within Implementation README) + +The table of contents within an implementation's `README.md` should contain one row for each submitted report and include the following columns: + +* **Extension Channel**: The channel of the Gateway API Inference Extension (e.g., `standard` or `experimental`). Must correspond to the `gatewayAPIInferenceExtensionChannel` field in the report. +* **Extension Version Tested**: The version of the Gateway API Inference Extension specification tested against (e.g., `v0.3.0`). Must correspond to the `gatewayAPIInferenceExtensionVersion` field in the report. +* **Implementation Version**: A link to the GitHub/website page for the specific release/commit of the implementation tested. The version value MUST correspond to the `implementation.version` field in the report. +* **Mode**: The operating mode of the implementation used for the test run (default is `default`). Must correspond to the `mode` field in the report. If a mode other than `default` is used, the "Reproduce" section must explain how to configure it. +* **Report**: A link to the corresponding report YAML file. Reports MUST be named according to the pattern: `---report.yaml`. + +### Reproduce Section (within Implementation README) + +This section MUST exist and contain the manual or automatic steps required to reproduce the results claimed by the uploaded conformance reports for that specific implementation. If reproduction steps differ significantly between implementation versions, use sub-sections. + +## Report Files + +Conformance reports MUST be uploaded exactly as generated by the official Gateway API Inference Extension conformance test suite, without any modifications. The "Reproduce" section allows for verification of the submitted report against a fresh run. + +### Report Rules + +To be accepted, submitted conformance reports must comply with the following rules: + +1. **Implementation Details:** All fields within the `implementation` block must have meaningful values: + * `organization`: The entity maintaining the implementation (company, open source org, individual). + * `project`: The name of the implementation project, unique within the organization. + * `url`: A valid URL for the project (e.g., GitHub repository, product page). + * `version`: A specific, reproducible snapshot of the implementation (e.g., tag, commit hash, release version). Branch names are not acceptable. + * `contact`: A list of contact points (GitHub handles like `@maintainer`, team handles like `@org/team`, email addresses, or support URLs like an issue tracker). +2. **Inference Extension Versioning:** The report MUST include: + * `gatewayAPIInferenceExtensionVersion`: The specific version of the Gateway API Inference Extension specification tested against (e.g., `v0.3.0`). + * `gatewayAPIInferenceExtensionChannel`: The channel (`standard` or `experimental`) corresponding to the tested version. +3. **Mode:** The `mode` field indicates the implementation's operating mode during the test run. +4. **Test Profile & Result:** + * The report MUST contain exactly one profile result under the `profiles` list. + * This profile MUST have `name: InferenceGateway`. + * The profile's `result` field MUST be `success`. A `success` result indicates that **all** tests defined within the Gateway API Inference Extension conformance suite for the specified version passed. (Note: Unlike core Gateway API, there are currently no optional "extended" features within the Inference Extension conformance profile itself; all included tests are considered mandatory for conformance). + +## Submission Process + +Conformance reports demonstrating a `success` result for the `InferenceGateway` profile should be submitted via Pull Request directly to this repository (`kubernetes-sigs/gateway-api-inference-extension`). + +1. Create a new folder under `/conformance/reports//` named after your implementation (e.g., `/conformance/reports/v0.3.0/my-implementation/`). +2. Add your implementation's `README.md` to this folder, following the structure described above. +3. Add your generated conformance report YAML file(s) to this folder, ensuring they follow the naming convention `---report.yaml`. +4. Submit the Pull Request. \ No newline at end of file diff --git a/conformance/resources/manifests/manifests.yaml b/conformance/resources/manifests/manifests.yaml new file mode 100644 index 00000000..acbb5197 --- /dev/null +++ b/conformance/resources/manifests/manifests.yaml @@ -0,0 +1,104 @@ +# Base Kubernetes resources for the Gateway API Inference Extension conformance tests. +# This includes namespaces and a minimal set of resources (Gateway, Backend) +# required by many tests. More specific resources should be defined within +# individual test files or other resource directories (e.g., sample_backends). + +--- +# Namespace for core infrastructure like Gateways. +apiVersion: v1 +kind: Namespace +metadata: + name: gateway-conformance-infra + labels: + gateway-conformance: infra + +--- +# Namespace for application backends (potentially simulating model servers +# or where InferencePools might reside in some tests). +apiVersion: v1 +kind: Namespace +metadata: + name: gateway-conformance-app-backend + labels: + gateway-conformance: backend + +--- +# A basic Gateway resource that allows HTTPRoutes from the same namespace. +# Tests can use this as a parent reference for routes that target InferencePools. +apiVersion: gateway.networking.k8s.io/v1 # Using v1 as per latest Gateway API standard +kind: Gateway +metadata: + name: same-namespace + namespace: gateway-conformance-infra +spec: + # The conformance suite runner will replace this placeholder + # with the actual GatewayClass name provided via flags. + gatewayClassName: "{GATEWAY_CLASS_NAME}" + listeners: + - name: http # Standard listener name + port: 80 + protocol: HTTP + allowedRoutes: + namespaces: + from: Same # Restrict to same namespace initially for simplicity + kinds: + # Allows HTTPRoutes to attach, which can then reference InferencePools. + - group: gateway.networking.k8s.io + kind: HTTPRoute +--- +# A simple backend service (using the standard echo server image) +# in the infra namespace. Can be used for basic connectivity checks +# or as a default backend if needed. +apiVersion: v1 +kind: Service +metadata: + name: infra-backend-v1 + namespace: gateway-conformance-infra +spec: + ports: + - name: http # Port name matches common usage + port: 8080 # Standard port for backend examples + protocol: TCP + targetPort: 3000 # Default port the echo-basic image listens on + selector: + app: infra-backend-v1 +--- +# Deployment for the simple backend service. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: infra-backend-v1 + namespace: gateway-conformance-infra + labels: + app: infra-backend-v1 +spec: + replicas: 1 + selector: + matchLabels: + app: infra-backend-v1 + template: + metadata: + labels: + app: infra-backend-v1 + spec: + containers: + - name: infra-backend-v1 + # Using the same image as core Gateway API tests for consistency. + image: gcr.io/k8s-staging-gateway-api/echo-basic:v20240412-v1.0.0-394-g40c666fd + ports: + - containerPort: 3000 + name: http + protocol: TCP + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + resources: + requests: + cpu: 10m + memory: 10Mi \ No newline at end of file diff --git a/conformance/tests/basic/inferencepool_accepted.go b/conformance/tests/basic/inferencepool_accepted.go new file mode 100644 index 00000000..d3d446a8 --- /dev/null +++ b/conformance/tests/basic/inferencepool_accepted.go @@ -0,0 +1,76 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package basic + +import ( + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" // For standard condition types + "sigs.k8s.io/gateway-api/conformance/utils/suite" + "sigs.k8s.io/gateway-api/pkg/features" // For standard feature names + + // Import the tests package to append to ConformanceTests + "sigs.k8s.io/gateway-api-inference-extension/conformance/tests" +) + +func init() { + // Register the InferencePoolAccepted test case with the conformance suite. + // This ensures it will be discovered and run by the test runner. + tests.ConformanceTests = append(tests.ConformanceTests, InferencePoolAccepted) +} + +// InferencePoolAccepted defines the test case for verifying basic InferencePool acceptance. +var InferencePoolAccepted = suite.ConformanceTest{ + ShortName: "InferencePoolAccepted", + Description: "A minimal InferencePool resource should be accepted by the controller", + Manifests: []string{"tests/basic/inferencepool_accepted.yaml"}, + Features: []features.FeatureName{}, + Test: func(t *testing.T, s *suite.ConformanceTestSuite) { + // created by the associated manifest file. + poolNN := types.NamespacedName{Name: "inferencepool-basic-accepted", Namespace: "gateway-conformance-infra"} + + t.Run("InferencePool should have Accepted condition set to True", func(t *testing.T) { + // Define the expected status condition. We use the standard "Accepted" + // condition type from the Gateway API for consistency. + acceptedCondition := metav1.Condition{ + Type: string(gatewayv1.GatewayConditionAccepted), // Standard condition type + Status: metav1.ConditionTrue, + Reason: "", // "" means we don't strictly check the Reason for this basic test. + } + InferencePoolMustHaveCondition(t, s, poolNN, acceptedCondition) + }) + }, +} + +// InferencePoolMustHaveCondition waits for the specified InferencePool resource +// to exist and report the expected status condition. +// TODO: move this helper function in your conformance/utils/kubernetes package. +// It should fetch the InferencePool using the provided client and check its +// Status.Conditions field, polling until the condition is met or a timeout occurs. +func InferencePoolMustHaveCondition(t *testing.T, s *suite.ConformanceTestSuite, poolNN types.NamespacedName, expectedCondition metav1.Condition) { + t.Helper() + + // Placeholder implementation - This needs to be replaced with actual logic. + t.Logf("Verification for InferencePool condition (%s=%s) on %s - Placeholder: Skipping check.", + expectedCondition.Type, expectedCondition.Status, poolNN.String()) + + // Skip the test for now until the helper is implemented. + // This allows the rest of the suite setup to proceed during initial development. + t.Skip("InferencePoolMustHaveCondition helper not yet implemented") +} diff --git a/conformance/tests/basic/inferencepool_accepted.yaml b/conformance/tests/basic/inferencepool_accepted.yaml new file mode 100644 index 00000000..acd442bf --- /dev/null +++ b/conformance/tests/basic/inferencepool_accepted.yaml @@ -0,0 +1,26 @@ +# Basic InferencePool for acceptance testing. +# This manifest defines the minimal required fields to create a valid +# InferencePool resource, which the InferencePoolAccepted test will use +# to verify that the controller recognizes and accepts the resource. + +apiVersion: inference.gateway.networking.k8s.io/v1alpha1 +kind: InferencePool +metadata: + # This name must match the 'poolNN' variable defined in the + # conformance/tests/basic/inferencepool_accepted.go test file. + name: inferencepool-basic-accepted + # This namespace should be one created by the base manifests. + namespace: gateway-conformance-infra +spec: + # --- Minimal Required Fields --- + # TODO: Replace these example fields with the *actual* minimum required + # fields according to the InferencePool CRD definition. + + # Example: Assuming 'replicas' is a required field. + replicas: 1 + + template: + spec: + modelFormat: + name: "example-minimal-format" + diff --git a/conformance/tests/main.go b/conformance/tests/main.go new file mode 100644 index 00000000..fc66c765 --- /dev/null +++ b/conformance/tests/main.go @@ -0,0 +1,35 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package tests is the root package for all Gateway API Inference Extension +// conformance test implementations. +package tests + +import ( + // Importing the suite package to access the ConformanceTest struct definition. + // For initial version directly importing from the core gateway-api repo. + // This may be adjusted in the future if we have need to create a copy of + // the suite utilities. + "sigs.k8s.io/gateway-api/conformance/utils/suite" + // Do NOT add blank imports for specific test packages here. + // They should be added to the main conformance package instead + // to avoid import cycles. +) + +// ConformanceTests holds all the conformance tests definitions for the +// Gateway API Inference Extension suite. Tests are registered from other packages +// using init() functions like the one in the basic package. +var ConformanceTests []suite.ConformanceTest diff --git a/conformance/utils/assertions.go b/conformance/utils/assertions.go new file mode 100644 index 00000000..c77d0fc5 --- /dev/null +++ b/conformance/utils/assertions.go @@ -0,0 +1,25 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package assertions contains custom assertion helper functions used within +// the Gateway API Inference Extension conformance test suite. +package assertions + +// TODO: Implement custom assertion functions specific to Inference Extension testing. +// Examples might include: +// - Asserting specific fields or structures within an inference API response body. +// - Asserting specific metrics reported by mock model servers or EPPs. +// - Asserting specific conditions or status fields unique to InferencePool or InferenceModel. diff --git a/conformance/utils/kubernetes/helpers.go b/conformance/utils/kubernetes/helpers.go new file mode 100644 index 00000000..5ebadfaf --- /dev/null +++ b/conformance/utils/kubernetes/helpers.go @@ -0,0 +1,46 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package kubernetes contains helper functions for interacting with +// Kubernetes objects within the conformance test suite. +package kubernetes + +import ( + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/gateway-api/conformance/utils/suite" +) + +// InferencePoolMustHaveCondition waits for the specified InferencePool resource +// to exist and report the expected status condition. +// This is a placeholder and needs full implementation. +// +// TODO: Implement the actual logic for this helper function. +// It should fetch the InferencePool using the provided client and check its +// Status.Conditions field, polling until the condition is met or a timeout occurs. +// like HTTPRouteMustHaveCondition. +func InferencePoolMustHaveCondition(t *testing.T, s *suite.ConformanceTestSuite, poolNN types.NamespacedName, expectedCondition metav1.Condition) { + t.Helper() // Marks this function as a test helper + + // Placeholder implementation: Log and skip the check. + t.Logf("Verification for InferencePool condition (%s=%s) on %s - Placeholder: Skipping check.", + expectedCondition.Type, expectedCondition.Status, poolNN.String()) + + // Skip the test using this helper until it's fully implemented. + t.Skip("InferencePoolMustHaveCondition helper not yet implemented") +} diff --git a/conformance/utils/traffic/traffic.go b/conformance/utils/traffic/traffic.go new file mode 100644 index 00000000..4f13f980 --- /dev/null +++ b/conformance/utils/traffic/traffic.go @@ -0,0 +1,22 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package traffic contains helper functions specifically for generating, +// sending, and validating network traffic related to inference workloads +// within the Gateway API Inference Extension conformance tests. +package traffic + +// TODO: Add helpers for specific inference protocols or request patterns as needed. diff --git a/go.mod b/go.mod index 20cf017a..d9cc5cda 100644 --- a/go.mod +++ b/go.mod @@ -25,6 +25,7 @@ require ( k8s.io/component-base v0.32.3 k8s.io/utils v0.0.0-20241210054802-24370beab758 sigs.k8s.io/controller-runtime v0.20.4 + sigs.k8s.io/gateway-api v1.2.1 sigs.k8s.io/structured-merge-diff/v4 v4.6.0 sigs.k8s.io/yaml v1.4.0 ) @@ -42,17 +43,17 @@ require ( github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cncf/xds/go v0.0.0-20241223141626-cff3c89139a3 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect - github.com/emicklei/go-restful/v3 v3.11.0 // indirect + github.com/emicklei/go-restful/v3 v3.12.0 // indirect github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect github.com/evanphx/json-patch/v5 v5.9.11 // indirect - github.com/fatih/color v1.16.0 // indirect + github.com/fatih/color v1.17.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-logr/zapr v1.3.0 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect - github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/jsonreference v0.21.0 // indirect github.com/go-openapi/swag v0.23.0 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.0 // indirect @@ -67,10 +68,10 @@ require ( github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 // indirect github.com/google/uuid v1.6.0 // indirect - github.com/gorilla/websocket v1.5.0 // indirect + github.com/gorilla/websocket v1.5.1 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect github.com/huandu/xstrings v1.3.3 // indirect - github.com/imdario/mergo v0.3.11 // indirect + github.com/imdario/mergo v0.3.16 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -129,6 +130,6 @@ require ( k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f // indirect sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.0 // indirect - sigs.k8s.io/controller-tools v0.14.0 // indirect + sigs.k8s.io/controller-tools v0.16.3 // indirect sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect ) diff --git a/go.sum b/go.sum index cd6cd380..e8d24cd5 100644 --- a/go.sum +++ b/go.sum @@ -23,25 +23,24 @@ github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/cncf/xds/go v0.0.0-20241223141626-cff3c89139a3 h1:boJj011Hh+874zpIySeApCX4GeOjPl9qhRF3QuIZq+Q= github.com/cncf/xds/go v0.0.0-20241223141626-cff3c89139a3/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/elastic/crd-ref-docs v0.1.0 h1:Cr5kz89QB3Iuuj7dhAfLMApCrChEGAaIBTxGk/xuRKw= github.com/elastic/crd-ref-docs v0.1.0/go.mod h1:X83mMBdJt05heJUYiS3T0yJ/JkCuliuhSUNav5Gjo/U= -github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= -github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/emicklei/go-restful/v3 v3.12.0 h1:y2DdzBAURM29NFF94q6RaY4vjIH1rtwDapwQtU84iWk= +github.com/emicklei/go-restful/v3 v3.12.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane/envoy v1.32.4 h1:jb83lalDRZSpPWW2Z7Mck/8kXZ5CQAFYVjQcdVIr83A= github.com/envoyproxy/go-control-plane/envoy v1.32.4/go.mod h1:Gzjc5k8JcJswLjAx1Zm+wSYE20UrLtt7JZMWiWQXQEw= github.com/envoyproxy/protoc-gen-validate v1.2.1 h1:DEo3O99U8j4hBFwbJfrz9VtgcDfUKS7KJ7spH3d86P8= github.com/envoyproxy/protoc-gen-validate v1.2.1/go.mod h1:d/C80l/jxXLdfEIhX1W2TmLfsJ31lvEjwamM4DxlWXU= -github.com/evanphx/json-patch v0.5.2 h1:xVCHIVMUu1wtM/VkR9jVZ45N3FhZfYMMYGorLCR8P3k= -github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= +github.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ2tG6yudJd8LBksgI= +github.com/evanphx/json-patch v5.7.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU= github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM= -github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= -github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= +github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= +github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= @@ -55,12 +54,10 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= -github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= -github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= -github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= -github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ= +github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4= github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs= @@ -96,14 +93,14 @@ github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= -github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= +github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0= github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k= github.com/huandu/xstrings v1.3.3 h1:/Gcsuc1x8JVbJ9/rlye4xZnVAbEkGauT8lbebqcQws4= github.com/huandu/xstrings v1.3.3/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= -github.com/imdario/mergo v0.3.11 h1:3tnifQM4i+fbajXKBHXWEH+KvNHqojZ778UH75j3bGA= -github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= +github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= @@ -114,11 +111,8 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= @@ -294,7 +288,6 @@ gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= @@ -326,8 +319,10 @@ sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.0 h1:CPT0ExVicCzcp sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.0/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= sigs.k8s.io/controller-runtime v0.20.4 h1:X3c+Odnxz+iPTRobG4tp092+CvBU9UK0t/bRf+n0DGU= sigs.k8s.io/controller-runtime v0.20.4/go.mod h1:xg2XB0K5ShQzAgsoujxuKN4LNXR2LfwwHsPj7Iaw+XY= -sigs.k8s.io/controller-tools v0.14.0 h1:rnNoCC5wSXlrNoBKKzL70LNJKIQKEzT6lloG6/LF73A= -sigs.k8s.io/controller-tools v0.14.0/go.mod h1:TV7uOtNNnnR72SpzhStvPkoS/U5ir0nMudrkrC4M9Sc= +sigs.k8s.io/controller-tools v0.16.3 h1:z48C5/d4jCVQQvtiSBL5MYyZ3EO2eFIOXrIKMgHVhFY= +sigs.k8s.io/controller-tools v0.16.3/go.mod h1:AEj6k+w1kYpLZv2einOH3mj52ips4W/6FUjnB5tkJGs= +sigs.k8s.io/gateway-api v1.2.1 h1:fZZ/+RyRb+Y5tGkwxFKuYuSRQHu9dZtbjenblleOLHM= +sigs.k8s.io/gateway-api v1.2.1/go.mod h1:EpNfEXNjiYfUJypf0eZ0P5iXA9ekSGWaS1WgPaM42X0= sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8= sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3/go.mod h1:18nIHnGi6636UCz6m8i4DhaJ65T6EruyzmoQqI2BVDo= sigs.k8s.io/randfill v0.0.0-20250304075658-069ef1bbf016 h1:kXv6kKdoEtedwuqMmkqhbkgvYKeycVbC8+iPCP9j5kQ= From 0ec5d143af50f10e8117c09db4908df6ec4b4645 Mon Sep 17 00:00:00 2001 From: Sina Chavoshi Date: Tue, 22 Apr 2025 19:15:20 +0000 Subject: [PATCH 02/15] fix missing go.sum entry --- go.sum | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/go.sum b/go.sum index b7fe4db8..981344c3 100644 --- a/go.sum +++ b/go.sum @@ -111,7 +111,6 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -328,7 +327,7 @@ sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1 sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3/go.mod h1:18nIHnGi6636UCz6m8i4DhaJ65T6EruyzmoQqI2BVDo= sigs.k8s.io/randfill v0.0.0-20250304075658-069ef1bbf016 h1:kXv6kKdoEtedwuqMmkqhbkgvYKeycVbC8+iPCP9j5kQ= sigs.k8s.io/randfill v0.0.0-20250304075658-069ef1bbf016/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= -sigs.k8s.io/structured-merge-diff/v4 v4.7.0 h1:qPeWmscJcXP0snki5IYF79Z8xrl8ETFxgMd7wez1XkI= -sigs.k8s.io/structured-merge-diff/v4 v4.7.0/go.mod h1:dDy58f92j70zLsuZVuUX5Wp9vtxXpaZnkPGWeqDfCps= +sigs.k8s.io/structured-merge-diff/v4 v4.6.0 h1:IUA9nvMmnKWcj5jl84xn+T5MnlZKThmUW1TdblaLVAc= +sigs.k8s.io/structured-merge-diff/v4 v4.6.0/go.mod h1:dDy58f92j70zLsuZVuUX5Wp9vtxXpaZnkPGWeqDfCps= sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= From cfd0a2a1c0629534b17e08f32b83e0fc312c9cb5 Mon Sep 17 00:00:00 2001 From: Sina Chavoshi Date: Tue, 22 Apr 2025 21:38:46 +0000 Subject: [PATCH 03/15] Fix the API version and basic Inferencepool-basic-accepted Yaml definition. --- conformance/conformance.go | 11 +++++--- .../tests/basic/inferencepool_accepted.go | 2 +- .../tests/basic/inferencepool_accepted.yaml | 25 ++++++++++--------- 3 files changed, 21 insertions(+), 17 deletions(-) diff --git a/conformance/conformance.go b/conformance/conformance.go index e1583520..44aa4531 100644 --- a/conformance/conformance.go +++ b/conformance/conformance.go @@ -53,6 +53,9 @@ import ( _ "sigs.k8s.io/gateway-api-inference-extension/conformance/tests/basic" // TODO: Add blank imports for other test categories as they are created. // _ "sigs.k8s.io/gateway-api-inference-extension/conformance/tests/model_routing" + + // == Import the Inference Extension API types == + inferencev1alpha2 "sigs.k8s.io/gateway-api-inference-extension/api/v1alpha2" ) // ConformanceProfileName defines the name for the Inference Extension conformance profile. @@ -87,9 +90,9 @@ func DefaultOptions(t *testing.T) confsuite.ConformanceOptions { } // Register necessary API Types - require.NoError(t, gatewayv1.AddToScheme(scheme)) // Add core Gateway API types - // TODO: Add the Inference Extension API types to the scheme - // Example: require.NoError(t, inferencev1alpha1.AddToScheme(scheme)) + require.NoError(t, gatewayv1.Install(scheme)) // Add core Gateway API types + // Add the Inference Extension API types to the scheme using the correct import alias + require.NoError(t, inferencev1alpha2.Install(scheme)) require.NoError(t, apiextensionsv1.AddToScheme(scheme)) // Needed for CRD checks // Create the Kubernetes clients @@ -120,7 +123,7 @@ func DefaultOptions(t *testing.T) confsuite.ConformanceOptions { // --- Inference Extension Specific Report Fields --- // TODO: Determine the source for these values. inferenceExtensionChannel := "experimental" - inferenceExtensionVersion := "v0.3.0" + inferenceExtensionVersion := "v1alpha2" _ = inferenceExtensionChannel // Avoid unused variable error until implemented _ = inferenceExtensionVersion // Avoid unused variable error until implemented diff --git a/conformance/tests/basic/inferencepool_accepted.go b/conformance/tests/basic/inferencepool_accepted.go index d3d446a8..8c61a385 100644 --- a/conformance/tests/basic/inferencepool_accepted.go +++ b/conformance/tests/basic/inferencepool_accepted.go @@ -43,7 +43,7 @@ var InferencePoolAccepted = suite.ConformanceTest{ Features: []features.FeatureName{}, Test: func(t *testing.T, s *suite.ConformanceTestSuite) { // created by the associated manifest file. - poolNN := types.NamespacedName{Name: "inferencepool-basic-accepted", Namespace: "gateway-conformance-infra"} + poolNN := types.NamespacedName{Name: "inferencepool-basic-accepted", Namespace: "default"} t.Run("InferencePool should have Accepted condition set to True", func(t *testing.T) { // Define the expected status condition. We use the standard "Accepted" diff --git a/conformance/tests/basic/inferencepool_accepted.yaml b/conformance/tests/basic/inferencepool_accepted.yaml index acd442bf..3fdfe95f 100644 --- a/conformance/tests/basic/inferencepool_accepted.yaml +++ b/conformance/tests/basic/inferencepool_accepted.yaml @@ -3,24 +3,25 @@ # InferencePool resource, which the InferencePoolAccepted test will use # to verify that the controller recognizes and accepts the resource. -apiVersion: inference.gateway.networking.k8s.io/v1alpha1 +apiVersion: inference.networking.x-k8s.io/v1alpha2 kind: InferencePool metadata: # This name must match the 'poolNN' variable defined in the # conformance/tests/basic/inferencepool_accepted.go test file. name: inferencepool-basic-accepted # This namespace should be one created by the base manifests. - namespace: gateway-conformance-infra + namespace: default spec: - # --- Minimal Required Fields --- - # TODO: Replace these example fields with the *actual* minimum required - # fields according to the InferencePool CRD definition. + # --- Selector (Required) --- + # Selects the Pods belonging to this pool. + selector: + app: "vllm-llama3-8b-instruct" - # Example: Assuming 'replicas' is a required field. - replicas: 1 - - template: - spec: - modelFormat: - name: "example-minimal-format" + # --- Target Port (Required) --- + # The port the model server container listens on. + targetPortNumber: 8000 + # --- Extension Reference --- + # GKE-specific configuration reference. + extensionRef: + name: vllm-llama3-8b-instruct-epp \ No newline at end of file From c88f0c34793824343bc7055c2be9dc8a2b5fbacc Mon Sep 17 00:00:00 2001 From: Sina Chavoshi Date: Tue, 22 Apr 2025 22:12:27 +0000 Subject: [PATCH 04/15] exclude conformance tests from github acceptance test run. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index a1845560..fe5ae14e 100644 --- a/Makefile +++ b/Makefile @@ -121,7 +121,7 @@ vet: ## Run go vet against code. .PHONY: test test: manifests generate fmt vet envtest image-build ## Run tests. - KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e) -race -coverprofile cover.out + KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e | grep -v /conformance) -race -coverprofile cover.out .PHONY: test-integration test-integration: ## Run tests. From a090edbff117edd24cf705d95b4da7fc5f8da7cb Mon Sep 17 00:00:00 2001 From: Sina Chavoshi Date: Fri, 25 Apr 2025 18:09:42 +0000 Subject: [PATCH 05/15] Add support for multiple profiles, remove release channel and update version to use semver. --- conformance/conformance.go | 48 +++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 22 deletions(-) diff --git a/conformance/conformance.go b/conformance/conformance.go index 44aa4531..326e6904 100644 --- a/conformance/conformance.go +++ b/conformance/conformance.go @@ -47,27 +47,31 @@ import ( // Import the test definitions package to access the ConformanceTests slice "sigs.k8s.io/gateway-api-inference-extension/conformance/tests" - // == Import test packages using blank identifier == + // Import test packages using blank identifier // This triggers the init() functions in these packages, which register the tests // by appending them to the tests.ConformanceTests slice. _ "sigs.k8s.io/gateway-api-inference-extension/conformance/tests/basic" // TODO: Add blank imports for other test categories as they are created. // _ "sigs.k8s.io/gateway-api-inference-extension/conformance/tests/model_routing" - // == Import the Inference Extension API types == + // Import the Inference Extension API types inferencev1alpha2 "sigs.k8s.io/gateway-api-inference-extension/api/v1alpha2" ) -// ConformanceProfileName defines the name for the Inference Extension conformance profile. -const InferenceConformanceProfileName confsuite.ConformanceProfileName = "InferenceGateway" +// GatewayLayerProfileName defines the name for the conformance profile that tests +// the Gateway API layer aspects of the Inference Extension (e.g., InferencePool, InferenceModel CRDs). +// Future profiles will cover EPP and ModelServer layers. +const GatewayLayerProfileName confsuite.ConformanceProfileName = "Gateway" -var InferenceCoreFeatures = sets.New[features.FeatureName]() // Placeholder - Populate with actual features +var InferenceCoreFeatures = sets.New[features.FeatureName]() // Placeholder - Populate with actual features specific to this profile or manage features per profile -// Define the Inference Gateway conformance profile. -// As per the design doc, initially, there are no "extended" features; -// all tested features are considered mandatory for this profile. -var InferenceGatewayProfile = confsuite.ConformanceProfile{ - Name: InferenceConformanceProfileName, +// GatewayLayerProfile defines the conformance profile for the Gateway API layer +// of the Inference Extension. +// In future iterations, we will add constants and ConformanceProfile structs for +// EPPProfileName ("EPP") and ModelServerProfileName ("ModelServer") +// to cover their respective conformance layers. +var GatewayLayerProfile = confsuite.ConformanceProfile{ + Name: GatewayLayerProfileName, CoreFeatures: InferenceCoreFeatures, } @@ -108,8 +112,9 @@ func DefaultOptions(t *testing.T) confsuite.ConformanceOptions { skipTests := confsuite.ParseSkipTests(*confflags.SkipTests) namespaceLabels := confsuite.ParseKeyValuePairs(*confflags.NamespaceLabels) namespaceAnnotations := confsuite.ParseKeyValuePairs(*confflags.NamespaceAnnotations) - // For this suite, we only run the InferenceGateway profile - conformanceProfiles := sets.New(InferenceConformanceProfileName) + // Initially, run the GatewayLayerProfile. This will expand as other profiles + // (EPP, ModelServer) are added and can be selected via flags in future iterations. + conformanceProfiles := sets.New(GatewayLayerProfileName) // Implementation details from flags implementation := confsuite.ParseImplementation( @@ -120,11 +125,8 @@ func DefaultOptions(t *testing.T) confsuite.ConformanceOptions { *confflags.ImplementationContact, ) - // --- Inference Extension Specific Report Fields --- - // TODO: Determine the source for these values. - inferenceExtensionChannel := "experimental" - inferenceExtensionVersion := "v1alpha2" - _ = inferenceExtensionChannel // Avoid unused variable error until implemented + // Inference Extension Specific Report Fields + inferenceExtensionVersion := "v0.3.0" _ = inferenceExtensionVersion // Avoid unused variable error until implemented // Create ConformanceOptions @@ -154,10 +156,10 @@ func DefaultOptions(t *testing.T) confsuite.ConformanceOptions { // GatewayAPIInferenceExtensionVersion: inferenceExtensionVersion, } - // Populate SupportedFeatures based on the InferenceGateway profile. + // Populate SupportedFeatures based on the GatewayLayerProfile. // Since all features are mandatory for this profile, add all defined core features. - if opts.ConformanceProfiles.Has(InferenceConformanceProfileName) { - for feature := range InferenceGatewayProfile.CoreFeatures { + if opts.ConformanceProfiles.Has(GatewayLayerProfileName) { + for feature := range GatewayLayerProfile.CoreFeatures { opts.SupportedFeatures.Insert(feature) } } @@ -179,8 +181,10 @@ func RunConformance(t *testing.T) { func RunConformanceWithOptions(t *testing.T, opts confsuite.ConformanceOptions) { t.Logf("Running Inference Extension conformance tests with GatewayClass %s", opts.GatewayClassName) - // Register the InferenceGateway profile with the suite runner. - confsuite.RegisterConformanceProfile(InferenceGatewayProfile) + // Register the GatewayLayerProfile with the suite runner. + // In the future, other profiles (EPP, ModelServer) will also be registered here, + // and the suite runner will execute tests based on the selected profiles. + confsuite.RegisterConformanceProfile(GatewayLayerProfile) // Initialize the test suite. cSuite, err := confsuite.NewConformanceTestSuite(opts) From 0d00b0f4c2b5033b9c67dc8b224895e193a857e2 Mon Sep 17 00:00:00 2001 From: Sina Chavoshi Date: Fri, 25 Apr 2025 18:34:20 +0000 Subject: [PATCH 06/15] Adding another layer to the report hierarchy for category of conformance (gateway, epp, model server). --- conformance/reports/README.md | 53 ++++++++++++++++++++--------------- 1 file changed, 30 insertions(+), 23 deletions(-) diff --git a/conformance/reports/README.md b/conformance/reports/README.md index a16c8014..e49b3b28 100644 --- a/conformance/reports/README.md +++ b/conformance/reports/README.md @@ -1,41 +1,47 @@ # Conformance Reports for Gateway API Inference Extension -This directory stores conformance reports submitted by various implementations of the Gateway API Inference Extension. This structure closely follows the [kubernetes-sigs/gateway-api/conformance/reports](https://github.com/kubernetes-sigs/gateway-api/blob/main/conformance/reports/README). +This directory stores conformance reports submitted by various implementations of the Gateway API Inference Extension. This structure closely follows the [kubernetes-sigs/gateway-api/conformance/reports](https://github.com/kubernetes-sigs/gateway-api/blob/main/conformance/reports/README.md). ## How this folder is structured -This folder stores conformance reports organized by the version of the Gateway API Inference Extension specification they were tested against: +This folder stores conformance reports organized first by the version of the Gateway API Inference Extension specification they were tested against, and then by the specific conformance profile (e.g., Gateway, EPP, Model Server): |-- conformance/reports -| |-- v0.3.0 # Example version -| | |-- my-inference-gateway -| | | |-- README.md -| | | |-- experimental-v1.2.3-default-report.yaml # Example report file -| | |-- another-implementation -| | | |-- README.md +| |-- v0.3.0 # Example extension version +| | |-- Gateway # Conformance profile/category +| | | |-- my-inference-gateway +| | | | |-- README.md +| | | | |-- experimental-v1.2.3-default-gateway-report.yaml # Example report file +| | | |-- another-implementation +| | | | |-- README.md +| | | | |-- ... +| | |-- EPP # Future conformance profile/category +| | | |-- my-epp-implementation +| | | | |-- ... +| | |-- ModelServer # Future conformance profile/category | | | |-- ... -| |-- v0.4.0 # Future version +| |-- v0.4.0 # Future extension version | | |-- ... ## Implementation Submissions -Each implementation conformant with a specific version of the Gateway API Inference Extension should have its own folder within the corresponding version directory (e.g., `/conformance/reports/v0.3.0/my-implementation/`). +Each implementation conformant with a specific profile of a specific version of the Gateway API Inference Extension should have its own folder within the corresponding version and profile directory (e.g., `/conformance/reports/v0.3.0/Gateway/my-implementation/`). The implementation is the owner of its folder and is responsible for: 1. Uploading one or more conformance reports (YAML files). 2. Maintaining a mandatory `README.md` file within their folder, structured as follows: - # My Inference Gateway Implementation + # My Inference Gateway Implementation (Gateway Profile Conformance) General information about the My/Implementation project. ## Table of Contents - | Extension Channel | Extension Version Tested | Implementation Version | Mode | Report | - |-------------------|--------------------------|------------------------|---------|---------------------------------------------------| - | experimental | v0.3.0 | v1.2.3 | default | [v1.2.3 report](./experimental-v1.2.3-default-report.yaml) | - | ... | ... | ... | ... | ... | + | Extension Channel | Extension Version Tested | Profile Tested | Implementation Version | Mode | Report | + |-------------------|--------------------------|----------------|------------------------|---------|---------------------------------------------------------------------| + | experimental | v0.3.0 | Gateway | v1.2.3 | default | [v1.2.3 Gateway report](./experimental-v1.2.3-default-gateway-report.yaml) | + | ... | ... | ... | ... | ... | ... | ## Reproduce @@ -47,9 +53,10 @@ The table of contents within an implementation's `README.md` should contain one * **Extension Channel**: The channel of the Gateway API Inference Extension (e.g., `standard` or `experimental`). Must correspond to the `gatewayAPIInferenceExtensionChannel` field in the report. * **Extension Version Tested**: The version of the Gateway API Inference Extension specification tested against (e.g., `v0.3.0`). Must correspond to the `gatewayAPIInferenceExtensionVersion` field in the report. +* **Profile Tested**: The specific conformance profile tested (e.g., `Gateway`, `EPP`, `ModelServer`). Must correspond to the `name` of the profile in the `profiles` list within the report. * **Implementation Version**: A link to the GitHub/website page for the specific release/commit of the implementation tested. The version value MUST correspond to the `implementation.version` field in the report. * **Mode**: The operating mode of the implementation used for the test run (default is `default`). Must correspond to the `mode` field in the report. If a mode other than `default` is used, the "Reproduce" section must explain how to configure it. -* **Report**: A link to the corresponding report YAML file. Reports MUST be named according to the pattern: `---report.yaml`. +* **Report**: A link to the corresponding report YAML file. Reports MUST be named according to the pattern: `----report.yaml` (e.g., `experimental-v1.2.3-default-gateway-report.yaml`). ### Reproduce Section (within Implementation README) @@ -74,15 +81,15 @@ To be accepted, submitted conformance reports must comply with the following rul * `gatewayAPIInferenceExtensionChannel`: The channel (`standard` or `experimental`) corresponding to the tested version. 3. **Mode:** The `mode` field indicates the implementation's operating mode during the test run. 4. **Test Profile & Result:** - * The report MUST contain exactly one profile result under the `profiles` list. - * This profile MUST have `name: InferenceGateway`. - * The profile's `result` field MUST be `success`. A `success` result indicates that **all** tests defined within the Gateway API Inference Extension conformance suite for the specified version passed. (Note: Unlike core Gateway API, there are currently no optional "extended" features within the Inference Extension conformance profile itself; all included tests are considered mandatory for conformance). + * The report MUST contain exactly one profile result under the `profiles` list for the specific conformance category being submitted (e.g., a report for "Gateway" conformance should only contain the "Gateway" profile result). + * The profile's `name` MUST match the conformance category (e.g., `Gateway`, `EPP`, `ModelServer`). + * The profile's `result` field MUST be `success`. A `success` result indicates that **all** tests defined within the Gateway API Inference Extension conformance suite for that specific profile and version passed. ## Submission Process -Conformance reports demonstrating a `success` result for the `InferenceGateway` profile should be submitted via Pull Request directly to this repository (`kubernetes-sigs/gateway-api-inference-extension`). +Conformance reports demonstrating a `success` result for a specific profile (e.g., `Gateway`) should be submitted via Pull Request directly to this repository (`kubernetes-sigs/gateway-api-inference-extension`). -1. Create a new folder under `/conformance/reports//` named after your implementation (e.g., `/conformance/reports/v0.3.0/my-implementation/`). +1. Create a new folder structure under `/conformance/reports///` named after your implementation (e.g., `/conformance/reports/v0.3.0/Gateway/my-implementation/`). 2. Add your implementation's `README.md` to this folder, following the structure described above. -3. Add your generated conformance report YAML file(s) to this folder, ensuring they follow the naming convention `---report.yaml`. -4. Submit the Pull Request. \ No newline at end of file +3. Add your generated conformance report YAML file(s) to this folder, ensuring they follow the naming convention `----report.yaml`. +4. Submit the Pull Request. From a0f600e07796d070dafeecf282b89a3b9031d39e Mon Sep 17 00:00:00 2001 From: Sina Chavoshi Date: Fri, 25 Apr 2025 19:26:15 +0000 Subject: [PATCH 07/15] Add trailing new line to yaml files. --- conformance/resources/manifests/manifests.yaml | 4 +++- conformance/tests/basic/inferencepool_accepted.yaml | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/conformance/resources/manifests/manifests.yaml b/conformance/resources/manifests/manifests.yaml index acbb5197..74ff4d78 100644 --- a/conformance/resources/manifests/manifests.yaml +++ b/conformance/resources/manifests/manifests.yaml @@ -46,6 +46,7 @@ spec: - group: gateway.networking.k8s.io kind: HTTPRoute --- + # A simple backend service (using the standard echo server image) # in the infra namespace. Can be used for basic connectivity checks # or as a default backend if needed. @@ -63,6 +64,7 @@ spec: selector: app: infra-backend-v1 --- + # Deployment for the simple backend service. apiVersion: apps/v1 kind: Deployment @@ -101,4 +103,4 @@ spec: resources: requests: cpu: 10m - memory: 10Mi \ No newline at end of file + memory: 10Mi diff --git a/conformance/tests/basic/inferencepool_accepted.yaml b/conformance/tests/basic/inferencepool_accepted.yaml index 3fdfe95f..fc826941 100644 --- a/conformance/tests/basic/inferencepool_accepted.yaml +++ b/conformance/tests/basic/inferencepool_accepted.yaml @@ -24,4 +24,4 @@ spec: # --- Extension Reference --- # GKE-specific configuration reference. extensionRef: - name: vllm-llama3-8b-instruct-epp \ No newline at end of file + name: vllm-llama3-8b-instruct-epp From 5201cba880bbce05ff3928865ec1db05f9cab7a3 Mon Sep 17 00:00:00 2001 From: Sina Chavoshi Date: Fri, 25 Apr 2025 19:40:48 +0000 Subject: [PATCH 08/15] switch to use InferencePoolMustHaveCondition from /conformance/utils/kubernetes --- .../tests/basic/inferencepool_accepted.go | 22 +++---------------- conformance/utils/kubernetes/helpers.go | 7 ++++-- 2 files changed, 8 insertions(+), 21 deletions(-) diff --git a/conformance/tests/basic/inferencepool_accepted.go b/conformance/tests/basic/inferencepool_accepted.go index 8c61a385..7b5d6130 100644 --- a/conformance/tests/basic/inferencepool_accepted.go +++ b/conformance/tests/basic/inferencepool_accepted.go @@ -27,6 +27,7 @@ import ( // Import the tests package to append to ConformanceTests "sigs.k8s.io/gateway-api-inference-extension/conformance/tests" + infrakubernetes "sigs.k8s.io/gateway-api-inference-extension/conformance/utils/kubernetes" ) func init() { @@ -38,7 +39,7 @@ func init() { // InferencePoolAccepted defines the test case for verifying basic InferencePool acceptance. var InferencePoolAccepted = suite.ConformanceTest{ ShortName: "InferencePoolAccepted", - Description: "A minimal InferencePool resource should be accepted by the controller", + Description: "A minimal InferencePool resource should be accepted by the controller and report an Accepted condition", Manifests: []string{"tests/basic/inferencepool_accepted.yaml"}, Features: []features.FeatureName{}, Test: func(t *testing.T, s *suite.ConformanceTestSuite) { @@ -53,24 +54,7 @@ var InferencePoolAccepted = suite.ConformanceTest{ Status: metav1.ConditionTrue, Reason: "", // "" means we don't strictly check the Reason for this basic test. } - InferencePoolMustHaveCondition(t, s, poolNN, acceptedCondition) + infrakubernetes.InferencePoolMustHaveCondition(t, s.Client, s.TimeoutConfig, poolNN, acceptedCondition) }) }, } - -// InferencePoolMustHaveCondition waits for the specified InferencePool resource -// to exist and report the expected status condition. -// TODO: move this helper function in your conformance/utils/kubernetes package. -// It should fetch the InferencePool using the provided client and check its -// Status.Conditions field, polling until the condition is met or a timeout occurs. -func InferencePoolMustHaveCondition(t *testing.T, s *suite.ConformanceTestSuite, poolNN types.NamespacedName, expectedCondition metav1.Condition) { - t.Helper() - - // Placeholder implementation - This needs to be replaced with actual logic. - t.Logf("Verification for InferencePool condition (%s=%s) on %s - Placeholder: Skipping check.", - expectedCondition.Type, expectedCondition.Status, poolNN.String()) - - // Skip the test for now until the helper is implemented. - // This allows the rest of the suite setup to proceed during initial development. - t.Skip("InferencePoolMustHaveCondition helper not yet implemented") -} diff --git a/conformance/utils/kubernetes/helpers.go b/conformance/utils/kubernetes/helpers.go index 5ebadfaf..3d517863 100644 --- a/conformance/utils/kubernetes/helpers.go +++ b/conformance/utils/kubernetes/helpers.go @@ -23,7 +23,10 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/gateway-api/conformance/utils/suite" + "sigs.k8s.io/controller-runtime/pkg/client" + + // Import necessary utilities from the core Gateway API conformance suite + "sigs.k8s.io/gateway-api/conformance/utils/config" ) // InferencePoolMustHaveCondition waits for the specified InferencePool resource @@ -34,7 +37,7 @@ import ( // It should fetch the InferencePool using the provided client and check its // Status.Conditions field, polling until the condition is met or a timeout occurs. // like HTTPRouteMustHaveCondition. -func InferencePoolMustHaveCondition(t *testing.T, s *suite.ConformanceTestSuite, poolNN types.NamespacedName, expectedCondition metav1.Condition) { +func InferencePoolMustHaveCondition(t *testing.T, c client.Client, timeoutConfig config.TimeoutConfig, poolNN types.NamespacedName, expectedCondition metav1.Condition) { t.Helper() // Marks this function as a test helper // Placeholder implementation: Log and skip the check. From 051e6c61c279e71c8e8d57e843562298b422dbe1 Mon Sep 17 00:00:00 2001 From: Sina Chavoshi Date: Fri, 25 Apr 2025 19:45:34 +0000 Subject: [PATCH 09/15] remove extra godoc comments --- conformance/conformance.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/conformance/conformance.go b/conformance/conformance.go index 326e6904..4b62f535 100644 --- a/conformance/conformance.go +++ b/conformance/conformance.go @@ -105,9 +105,6 @@ func DefaultOptions(t *testing.T) confsuite.ConformanceOptions { cs, err := clientset.NewForConfig(cfg) require.NoError(t, err, "error initializing Kubernetes clientset") - // Parse flags using the core Gateway API flag definitions - // supportedFeatures := confsuite.ParseSupportedFeatures(*confflags.SupportedFeatures) - // We derive this from the profile exemptFeatures := confsuite.ParseSupportedFeatures(*confflags.ExemptFeatures) skipTests := confsuite.ParseSkipTests(*confflags.SkipTests) namespaceLabels := confsuite.ParseKeyValuePairs(*confflags.NamespaceLabels) From 4ae363d87a65ead6f54b044c319817633ebe1bca Mon Sep 17 00:00:00 2001 From: Sina Chavoshi Date: Fri, 25 Apr 2025 20:09:55 +0000 Subject: [PATCH 10/15] Remove references to ExtensionChannel from reports readme --- conformance/reports/README.md | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/conformance/reports/README.md b/conformance/reports/README.md index e49b3b28..4e870987 100644 --- a/conformance/reports/README.md +++ b/conformance/reports/README.md @@ -38,10 +38,10 @@ The implementation is the owner of its folder and is responsible for: ## Table of Contents - | Extension Channel | Extension Version Tested | Profile Tested | Implementation Version | Mode | Report | - |-------------------|--------------------------|----------------|------------------------|---------|---------------------------------------------------------------------| - | experimental | v0.3.0 | Gateway | v1.2.3 | default | [v1.2.3 Gateway report](./experimental-v1.2.3-default-gateway-report.yaml) | - | ... | ... | ... | ... | ... | ... | +| Extension Version Tested | Profile Tested | Implementation Version | Mode | Report | +|--------------------------|----------------|------------------------|---------|---------------------------------------------------------------------| +| v0.3.0 | Gateway | v1.2.3 | default | [v1.2.3 Gateway report](./experimental-v1.2.3-default-gateway-report.yaml) | +| ... | ... | ... | ... | ... | ## Reproduce @@ -51,12 +51,11 @@ The implementation is the owner of its folder and is responsible for: The table of contents within an implementation's `README.md` should contain one row for each submitted report and include the following columns: -* **Extension Channel**: The channel of the Gateway API Inference Extension (e.g., `standard` or `experimental`). Must correspond to the `gatewayAPIInferenceExtensionChannel` field in the report. * **Extension Version Tested**: The version of the Gateway API Inference Extension specification tested against (e.g., `v0.3.0`). Must correspond to the `gatewayAPIInferenceExtensionVersion` field in the report. * **Profile Tested**: The specific conformance profile tested (e.g., `Gateway`, `EPP`, `ModelServer`). Must correspond to the `name` of the profile in the `profiles` list within the report. * **Implementation Version**: A link to the GitHub/website page for the specific release/commit of the implementation tested. The version value MUST correspond to the `implementation.version` field in the report. * **Mode**: The operating mode of the implementation used for the test run (default is `default`). Must correspond to the `mode` field in the report. If a mode other than `default` is used, the "Reproduce" section must explain how to configure it. -* **Report**: A link to the corresponding report YAML file. Reports MUST be named according to the pattern: `----report.yaml` (e.g., `experimental-v1.2.3-default-gateway-report.yaml`). +* **Report**: A link to the corresponding report YAML file. Reports MUST be named according to the pattern: `---report.yaml` (e.g., `experimental-v1.2.3-default-gateway-report.yaml`). ### Reproduce Section (within Implementation README) @@ -78,7 +77,6 @@ To be accepted, submitted conformance reports must comply with the following rul * `contact`: A list of contact points (GitHub handles like `@maintainer`, team handles like `@org/team`, email addresses, or support URLs like an issue tracker). 2. **Inference Extension Versioning:** The report MUST include: * `gatewayAPIInferenceExtensionVersion`: The specific version of the Gateway API Inference Extension specification tested against (e.g., `v0.3.0`). - * `gatewayAPIInferenceExtensionChannel`: The channel (`standard` or `experimental`) corresponding to the tested version. 3. **Mode:** The `mode` field indicates the implementation's operating mode during the test run. 4. **Test Profile & Result:** * The report MUST contain exactly one profile result under the `profiles` list for the specific conformance category being submitted (e.g., a report for "Gateway" conformance should only contain the "Gateway" profile result). @@ -91,5 +89,5 @@ Conformance reports demonstrating a `success` result for a specific profile (e.g 1. Create a new folder structure under `/conformance/reports///` named after your implementation (e.g., `/conformance/reports/v0.3.0/Gateway/my-implementation/`). 2. Add your implementation's `README.md` to this folder, following the structure described above. -3. Add your generated conformance report YAML file(s) to this folder, ensuring they follow the naming convention `----report.yaml`. +3. Add your generated conformance report YAML file(s) to this folder, ensuring they follow the naming convention `---report.yaml`. 4. Submit the Pull Request. From 4f117cc024a9ddc4cf3108831e3b39035ed8a891 Mon Sep 17 00:00:00 2001 From: Sina Chavoshi Date: Fri, 25 Apr 2025 20:10:26 +0000 Subject: [PATCH 11/15] format readme --- conformance/reports/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/conformance/reports/README.md b/conformance/reports/README.md index 4e870987..f20cf18c 100644 --- a/conformance/reports/README.md +++ b/conformance/reports/README.md @@ -38,10 +38,10 @@ The implementation is the owner of its folder and is responsible for: ## Table of Contents -| Extension Version Tested | Profile Tested | Implementation Version | Mode | Report | -|--------------------------|----------------|------------------------|---------|---------------------------------------------------------------------| +| Extension Version Tested | Profile Tested | Implementation Version | Mode | Report | +|--------------------------|----------------|------------------------|---------|----------------------------------------------------------------------------| | v0.3.0 | Gateway | v1.2.3 | default | [v1.2.3 Gateway report](./experimental-v1.2.3-default-gateway-report.yaml) | -| ... | ... | ... | ... | ... | +| ... | ... | ... | ... | ... | ## Reproduce From 0864dc113dbfbcfa1f75ec42456708467b4ea0c4 Mon Sep 17 00:00:00 2001 From: Sina Chavoshi Date: Fri, 25 Apr 2025 20:27:39 +0000 Subject: [PATCH 12/15] remove the service for the conformance backend. --- conformance/resources/manifests/manifests.yaml | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/conformance/resources/manifests/manifests.yaml b/conformance/resources/manifests/manifests.yaml index 74ff4d78..01f680f9 100644 --- a/conformance/resources/manifests/manifests.yaml +++ b/conformance/resources/manifests/manifests.yaml @@ -47,24 +47,6 @@ spec: kind: HTTPRoute --- -# A simple backend service (using the standard echo server image) -# in the infra namespace. Can be used for basic connectivity checks -# or as a default backend if needed. -apiVersion: v1 -kind: Service -metadata: - name: infra-backend-v1 - namespace: gateway-conformance-infra -spec: - ports: - - name: http # Port name matches common usage - port: 8080 # Standard port for backend examples - protocol: TCP - targetPort: 3000 # Default port the echo-basic image listens on - selector: - app: infra-backend-v1 ---- - # Deployment for the simple backend service. apiVersion: apps/v1 kind: Deployment From 595e7a73b87930555d154dd0bdf65f97d361fd68 Mon Sep 17 00:00:00 2001 From: Sina Chavoshi Date: Fri, 25 Apr 2025 22:00:44 +0000 Subject: [PATCH 13/15] update the namespace and EEP names to match the manifest. --- conformance/tests/basic/inferencepool_accepted.go | 2 +- conformance/tests/basic/inferencepool_accepted.yaml | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/conformance/tests/basic/inferencepool_accepted.go b/conformance/tests/basic/inferencepool_accepted.go index 7b5d6130..eae59404 100644 --- a/conformance/tests/basic/inferencepool_accepted.go +++ b/conformance/tests/basic/inferencepool_accepted.go @@ -44,7 +44,7 @@ var InferencePoolAccepted = suite.ConformanceTest{ Features: []features.FeatureName{}, Test: func(t *testing.T, s *suite.ConformanceTestSuite) { // created by the associated manifest file. - poolNN := types.NamespacedName{Name: "inferencepool-basic-accepted", Namespace: "default"} + poolNN := types.NamespacedName{Name: "inferencepool-basic-accepted", Namespace: "gateway-conformance-app-backend"} t.Run("InferencePool should have Accepted condition set to True", func(t *testing.T) { // Define the expected status condition. We use the standard "Accepted" diff --git a/conformance/tests/basic/inferencepool_accepted.yaml b/conformance/tests/basic/inferencepool_accepted.yaml index fc826941..8ae327d8 100644 --- a/conformance/tests/basic/inferencepool_accepted.yaml +++ b/conformance/tests/basic/inferencepool_accepted.yaml @@ -10,18 +10,18 @@ metadata: # conformance/tests/basic/inferencepool_accepted.go test file. name: inferencepool-basic-accepted # This namespace should be one created by the base manifests. - namespace: default + namespace: gateway-conformance-app-backend spec: # --- Selector (Required) --- # Selects the Pods belonging to this pool. selector: - app: "vllm-llama3-8b-instruct" + app: "infra-backend-v1" # --- Target Port (Required) --- # The port the model server container listens on. - targetPortNumber: 8000 + targetPortNumber: 3000 # --- Extension Reference --- # GKE-specific configuration reference. extensionRef: - name: vllm-llama3-8b-instruct-epp + name: infra-backend-v1-epp From a0d366afc107005a6b1ebc14f04d7586d95d379b Mon Sep 17 00:00:00 2001 From: Sina Chavoshi Date: Mon, 28 Apr 2025 18:40:44 +0000 Subject: [PATCH 14/15] Update PR based on review feedback including, change dir name to lower case, remove unused manifest, remove NamespaceLabels and NamespaceAnnotations --- conformance/conformance.go | 4 -- conformance/reports/README.md | 6 +-- .../resources/manifests/manifests.yaml | 41 ------------------- 3 files changed, 3 insertions(+), 48 deletions(-) diff --git a/conformance/conformance.go b/conformance/conformance.go index 4b62f535..20d80fde 100644 --- a/conformance/conformance.go +++ b/conformance/conformance.go @@ -107,8 +107,6 @@ func DefaultOptions(t *testing.T) confsuite.ConformanceOptions { exemptFeatures := confsuite.ParseSupportedFeatures(*confflags.ExemptFeatures) skipTests := confsuite.ParseSkipTests(*confflags.SkipTests) - namespaceLabels := confsuite.ParseKeyValuePairs(*confflags.NamespaceLabels) - namespaceAnnotations := confsuite.ParseKeyValuePairs(*confflags.NamespaceAnnotations) // Initially, run the GatewayLayerProfile. This will expand as other profiles // (EPP, ModelServer) are added and can be selected via flags in future iterations. conformanceProfiles := sets.New(GatewayLayerProfileName) @@ -140,8 +138,6 @@ func DefaultOptions(t *testing.T) confsuite.ConformanceOptions { ExemptFeatures: exemptFeatures, RunTest: *confflags.RunTest, Mode: *confflags.Mode, - NamespaceLabels: namespaceLabels, - NamespaceAnnotations: namespaceAnnotations, Implementation: implementation, ConformanceProfiles: conformanceProfiles, ManifestFS: []fs.FS{&Manifests}, // Assumes embed.go defines `Manifests` diff --git a/conformance/reports/README.md b/conformance/reports/README.md index f20cf18c..81652b1c 100644 --- a/conformance/reports/README.md +++ b/conformance/reports/README.md @@ -8,17 +8,17 @@ This folder stores conformance reports organized first by the version of the Gat |-- conformance/reports | |-- v0.3.0 # Example extension version -| | |-- Gateway # Conformance profile/category +| | |-- gateway # Conformance profile/category | | | |-- my-inference-gateway | | | | |-- README.md | | | | |-- experimental-v1.2.3-default-gateway-report.yaml # Example report file | | | |-- another-implementation | | | | |-- README.md | | | | |-- ... -| | |-- EPP # Future conformance profile/category +| | |-- epp # Future conformance profile/category | | | |-- my-epp-implementation | | | | |-- ... -| | |-- ModelServer # Future conformance profile/category +| | |-- model-server # Future conformance profile/category | | | |-- ... | |-- v0.4.0 # Future extension version | | |-- ... diff --git a/conformance/resources/manifests/manifests.yaml b/conformance/resources/manifests/manifests.yaml index 01f680f9..cb583694 100644 --- a/conformance/resources/manifests/manifests.yaml +++ b/conformance/resources/manifests/manifests.yaml @@ -45,44 +45,3 @@ spec: # Allows HTTPRoutes to attach, which can then reference InferencePools. - group: gateway.networking.k8s.io kind: HTTPRoute ---- - -# Deployment for the simple backend service. -apiVersion: apps/v1 -kind: Deployment -metadata: - name: infra-backend-v1 - namespace: gateway-conformance-infra - labels: - app: infra-backend-v1 -spec: - replicas: 1 - selector: - matchLabels: - app: infra-backend-v1 - template: - metadata: - labels: - app: infra-backend-v1 - spec: - containers: - - name: infra-backend-v1 - # Using the same image as core Gateway API tests for consistency. - image: gcr.io/k8s-staging-gateway-api/echo-basic:v20240412-v1.0.0-394-g40c666fd - ports: - - containerPort: 3000 - name: http - protocol: TCP - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - resources: - requests: - cpu: 10m - memory: 10Mi From b63ef3a8a23e802d3b06d5f18ffd35f98701aa6e Mon Sep 17 00:00:00 2001 From: Sina Chavoshi Date: Mon, 28 Apr 2025 20:25:48 +0000 Subject: [PATCH 15/15] add a comment to clarify use of echo server --- conformance/resources/manifests/manifests.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conformance/resources/manifests/manifests.yaml b/conformance/resources/manifests/manifests.yaml index cb583694..7b43b784 100644 --- a/conformance/resources/manifests/manifests.yaml +++ b/conformance/resources/manifests/manifests.yaml @@ -25,6 +25,8 @@ metadata: --- # A basic Gateway resource that allows HTTPRoutes from the same namespace. # Tests can use this as a parent reference for routes that target InferencePools. +# Using a simple echo server instead of an actual model server to simplify the test +# execution, this design may need to be revised based on the test case needs. apiVersion: gateway.networking.k8s.io/v1 # Using v1 as per latest Gateway API standard kind: Gateway metadata: