test: add LCOWV2 feature flag and v2 LCOW test surface

.github/workflows/ci.yml

@@ -36,7 +36,13 @@ jobs:
       - name: Run golangci-lint
         uses: golangci/golangci-lint-action@v9
         with:
-          version: v2.1
+          # v2.1.0 has a known typechecker bug that fails to resolve
+          # cross-module imports under `working-directory: test/` when those
+          # imports point back into the parent module via a `replace`
+          # directive. Symptom: `could not import ... (-: build constraints
+          # exclude all Go files in ...)` on `(windows, test)` despite the
+          # files matching the active build tags + GOOS. Fixed in v2.5+.
+          version: v2.5
           args: >-
             --verbose
             --max-issues-per-linter=0
@@ -383,6 +389,56 @@ jobs:
           exit $ec
         working-directory: test
 
+      # Run the v2 LCOW functional tests against the v2 vm.Controller
+      # end-to-end. The chokepoint in defaultLCOWOptions auto-skips every
+      # v1 LCOW test under -feature LCOWV2, so this invocation effectively
+      # runs only the TestLCOW_V2_* surface (and any future v2-aware tests).
+      # Marked continue-on-error: true while this surface is being grown so
+      # a v2-specific failure does not block the v1 pipeline.
+      #
+      # NOTE: Functional tests build HCS documents directly via
+      # internal/builder/vm/lcow and create v2 controllers in-process — they
+      # do NOT exercise the CRI -> containerd -> shim path. CRI v2 testing
+      # (Test_V2_LCOW_* in test/cri-containerd) requires a separate CI step
+      # that starts containerd with `snapshotter = "windows-lcow"` set on
+      # both `runhcs-lcow` AND `runhcs-lcow-v2` runtime blocks (the mapping
+      # logic lives in the containerd CRI plugin). That step is intentionally
+      # deferred to a follow-up PR alongside the integration-tests v2 setup.
+      - name: Build and run functional testing binary (LCOWV2)
+        continue-on-error: true
+        run: |
+          if ( -not (Test-Path './functional.test.exe') ) {
+            Write-Output '::warning::functional.test.exe missing; skipping LCOWV2 run'
+            exit 0
+          }
+
+          $gotestsum = Get-Command -Name 'gotestsum' -CommandType Application -ErrorAction 'Stop' |
+            Select-Object -First 1 -ExpandProperty Source
+          $go = Get-Command -Name 'go' -CommandType Application -ErrorAction Stop |
+            Select-Object -First 1 -ExpandProperty Source
+
+          # LCOWV2 implies LCOW in TestMain so featureLCOW-gated tests would
+          # also be reachable, but defaultLCOWOptions calls requireV1Only so
+          # they skip cleanly. Net effect: only TestLCOW_V2_* tests actually
+          # execute, exercising the v2 controller end-to-end.
+          $cmd = '${{ env.GOTESTSUM_CMD_RAW }} ./functional.test.exe -feature=LCOWV2 -exclude=LCOWIntegrity -test.timeout=1h -test.v -log-level=info'
+          $cmd = $cmd -replace '\bgo\b', $go
+          $cmd = $cmd -replace '\bgotestsum\b', $gotestsum
+          Write-Host "gotestsum command: $cmd"
+
+          psexec -nobanner -w (Get-Location) -s cmd /c "$cmd > v2-out.txt 2>&1"
+          $ec = $LASTEXITCODE
+
+          Get-Content v2-out.txt
+
+          exit $ec
+        working-directory: test
+
+      # Build the v2 LCOW shim binary so it is included in test_binaries
+      # artifact uploads and available to anyone reproducing v2 test runs.
+      - name: Build containerd-shim-lcow-v2 binary
+        run: ${{ env.GO_BUILD_CMD }} -tags lcow -o test/containerd-shim-lcow-v2.exe ./cmd/containerd-shim-lcow-v2
+
       # build testing binaries
       - name: Build cri-containerd Testing Binary
         run: ${{ env.GO_BUILD_TEST_CMD }} ./cri-containerd
@@ -400,6 +456,7 @@ jobs:
           name: test_binaries_${{ matrix.name }}
           path: |
             test/containerd-shim-runhcs-v1.test.exe
+            test/containerd-shim-lcow-v2.exe
             test/cri-containerd.test.exe
             test/functional.test.exe
             test/runhcs.test.exe

test/cri-containerd/container_layers_packing_test.go

@@ -46,6 +46,7 @@ func Test_Container_Layer_Packing_On_VPMem(t *testing.T) {
 	defer cancel()
 
 	requireFeatures(t, featureLCOW)
+	requireV1Only(t)
 
 	// use ubuntu to make sure that multiple container layers will be mapped properly
 	pullRequiredLCOWImages(t, []string{imageLcowK8sPause, ubuntu1804})
@@ -101,6 +102,7 @@ func Test_Many_Container_Layers_Supported_On_VPMem(t *testing.T) {
 	defer cancel()
 
 	requireFeatures(t, featureLCOW)
+	requireV1Only(t)
 
 	pullRequiredLCOWImages(t, []string{imageLcowK8sPause, alpine70ExtraLayers, ubuntu70ExtraLayers})
 
@@ -132,6 +134,7 @@ func Test_Annotation_Disable_Multi_Mapping(t *testing.T) {
 	defer cancel()
 
 	requireFeatures(t, featureLCOW)
+	requireV1Only(t)
 
 	pullRequiredLCOWImages(t, []string{imageLcowK8sPause, alpine70ExtraLayers})
 

test/cri-containerd/disable_vpmem_test.go

@@ -28,6 +28,7 @@ func uniqueRef() string {
 
 func Test_70LayerImagesWithNoVPmemForLayers(t *testing.T) {
 	requireFeatures(t, featureLCOW)
+	requireV1Only(t)
 
 	ubuntu70Image := "cplatpublic.azurecr.io/ubuntu70extra:18.04"
 	alpine70Image := "cplatpublic.azurecr.io/alpine70extra:latest"

test/cri-containerd/helper_sandbox_test.go

@@ -10,6 +10,42 @@ import (
 	runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
 )
 
+// isLCOWV2 reports whether the LCOWV2 feature flag is set on the current test
+// invocation. Callers should prefer the higher-level helpers below
+// (lcowRuntimeHandlerForTest, requireV1Only) so the V2 selection logic stays
+// in one place.
+func isLCOWV2() bool {
+	return flagFeatures.IsSet(featureLCOWV2)
+}
+
+// lcowRuntimeHandlerForTest returns the LCOW runtime handler that the current
+// test should target. When the LCOWV2 feature flag is set, it returns the V2
+// shim handler (containerd-shim-lcow-v2.exe via runtime_type
+// io.containerd.lcow.v2). Otherwise it returns the V1 handler
+// (containerd-shim-runhcs-v1.exe via runtime_type io.containerd.runhcs.v1).
+//
+// Tests that exercise generic LCOW lifecycle and work on both shims should use
+// this helper instead of hard-coding lcowRuntimeHandler, so the same suite can
+// be run twice in CI: once for V1 (default) and once with -feature LCOWV2 for V2.
+// Mirrors the pattern in the azcri repo.
+func lcowRuntimeHandlerForTest(tb testing.TB) string {
+	tb.Helper()
+	if isLCOWV2() {
+		return lcowV2RuntimeHandler
+	}
+	return lcowRuntimeHandler
+}
+
+// requireV1Only skips the test when the LCOWV2 feature flag is set.
+// Use this for tests that depend on V1-only features such as VPMEM,
+// VHD/initrd boot modes, or other UVM knobs not exposed in the v2 builder.
+func requireV1Only(tb testing.TB) {
+	tb.Helper()
+	if isLCOWV2() {
+		tb.Skip("test requires V1 shim features (VPMEM/VHD/initrd) not exposed in V2")
+	}
+}
+
 type SandboxConfigOpt func(*runtime.PodSandboxConfig) error
 
 func WithSandboxAnnotations(annotations map[string]string) SandboxConfigOpt {

test/cri-containerd/lcow_v2_test.go

@@ -0,0 +1,104 @@
+//go:build windows && functional
+// +build windows,functional
+
+// V2 LCOW-specific CRI tests. These mirror the Test_V2Sandbox_* pattern
+// established in the azcri test suite: each test gates on featureLCOWV2 and
+// targets lcowV2RuntimeHandler directly (no v1 fallback), because the
+// scenarios under test exercise the V2 runtime path.
+//
+// To run these tests:
+//  1. The CI containerd config must register `runhcs-lcow-v2` →
+//     `containerd-shim-lcow-v2.exe` (runtime_type io.containerd.lcow.v2).
+//  2. The test binary must be invoked with -feature=LCOWV2.
+
+package cri_containerd
+
+import (
+	"context"
+	"testing"
+
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
+)
+
+// Test_V2_LCOW_PodLifecycle exercises the basic pod-sandbox lifecycle through
+// the V2 shim: RunPodSandbox → StopPodSandbox → RemovePodSandbox. This is the
+// minimum end-to-end smoke test that proves containerd → shim handshake works
+// on the V2 path.
+func Test_V2_LCOW_PodLifecycle(t *testing.T) {
+	requireFeatures(t, featureLCOWV2)
+
+	pullRequiredLCOWImages(t, []string{imageLcowK8sPause})
+
+	client := newTestRuntimeClient(t)
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	podReq := getRunPodSandboxRequest(t, lcowV2RuntimeHandler)
+	podID := runPodSandbox(t, client, ctx, podReq)
+	defer removePodSandbox(t, client, ctx, podID)
+	defer stopPodSandbox(t, client, ctx, podID)
+}
+
+// Test_V2_LCOW_ContainerLifecycle exercises a full container lifecycle inside
+// a V2 sandbox: RunPodSandbox → CreateContainer → StartContainer →
+// StopContainer → RemoveContainer → StopPodSandbox → RemovePodSandbox.
+func Test_V2_LCOW_ContainerLifecycle(t *testing.T) {
+	requireFeatures(t, featureLCOWV2)
+
+	pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowAlpine})
+
+	client := newTestRuntimeClient(t)
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	podReq := getRunPodSandboxRequest(t, lcowV2RuntimeHandler)
+	podID := runPodSandbox(t, client, ctx, podReq)
+	defer removePodSandbox(t, client, ctx, podID)
+	defer stopPodSandbox(t, client, ctx, podID)
+
+	cReq := getCreateContainerRequest(podID, "alpine", imageLcowAlpine,
+		[]string{"echo", "hello"}, podReq.Config)
+	containerID := createContainer(t, client, ctx, cReq)
+	defer removeContainer(t, client, ctx, containerID)
+
+	startContainer(t, client, ctx, containerID)
+	stopContainer(t, client, ctx, containerID)
+}
+
+// Test_V2_LCOW_ContainerExec runs a workload container and verifies that
+// ExecSync into it succeeds with the expected exit code. Validates the GCS
+// exec path through the V2 controller.
+func Test_V2_LCOW_ContainerExec(t *testing.T) {
+	requireFeatures(t, featureLCOWV2)
+
+	pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowAlpine})
+
+	client := newTestRuntimeClient(t)
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	podReq := getRunPodSandboxRequest(t, lcowV2RuntimeHandler)
+	podID := runPodSandbox(t, client, ctx, podReq)
+	defer removePodSandbox(t, client, ctx, podID)
+	defer stopPodSandbox(t, client, ctx, podID)
+
+	cReq := getCreateContainerRequest(podID, "alpine", imageLcowAlpine,
+		[]string{"top"}, podReq.Config)
+	containerID := createContainer(t, client, ctx, cReq)
+	defer removeContainer(t, client, ctx, containerID)
+
+	startContainer(t, client, ctx, containerID)
+	defer stopContainer(t, client, ctx, containerID)
+
+	execResp, err := client.ExecSync(ctx, &runtime.ExecSyncRequest{
+		ContainerId: containerID,
+		Cmd:         []string{"echo", "hello"},
+		Timeout:     20,
+	})
+	if err != nil {
+		t.Fatalf("ExecSync failed: %v", err)
+	}
+	if execResp.ExitCode != 0 {
+		t.Fatalf("ExecSync returned exit code %d, stderr: %s", execResp.ExitCode, string(execResp.Stderr))
+	}
+}

test/cri-containerd/main_test.go

@@ -29,6 +29,7 @@ const (
 
 	// TODO: remove lcow when shim only tests are relocated
 	lcowRuntimeHandler                = "runhcs-lcow"
+	lcowV2RuntimeHandler              = "runhcs-lcow-v2"
 	wcowProcessRuntimeHandler         = "runhcs-wcow-process"
 	wcowHypervisorRuntimeHandler      = "runhcs-wcow-hypervisor"
 	wcowHypervisor17763RuntimeHandler = "runhcs-wcow-hypervisor-17763"
@@ -98,6 +99,7 @@ var (
 // Make sure you update allFeatures below with any new features you add.
 const (
 	featureLCOW               = "LCOW"
+	featureLCOWV2             = "LCOWV2"
 	featureWCOWProcess        = "WCOWProcess"
 	featureWCOWHypervisor     = "WCOWHypervisor"
 	featureHostProcess        = "HostProcess"
@@ -109,6 +111,7 @@ const (
 
 var allFeatures = []string{
 	featureLCOW,
+	featureLCOWV2,
 	featureWCOWProcess,
 	featureWCOWHypervisor,
 	featureHostProcess,
@@ -120,6 +123,15 @@ var allFeatures = []string{
 
 func TestMain(m *testing.M) {
 	flag.Parse()
+	// LCOWV2 implies LCOW: the v2 shim IS an LCOW runtime, so a run gated
+	// only on `-feature LCOWV2` should still execute tests that gate on
+	// `featureLCOW`. Without this implication, the same test suite would
+	// need parallel `-feature LCOW` and `-feature LCOWV2` invocations OR
+	// every LCOW-gated test would have to be rewritten to accept either
+	// flag. Mirrors the pattern established in the azcri test suite.
+	if flagFeatures.IncludesExplicit() && flagFeatures.IsSet(featureLCOWV2) {
+		flagFeatures.Include(featureLCOW)
+	}
 	os.Exit(m.Run())
 }
 
@@ -205,10 +217,23 @@ func pullRequiredLCOWImages(tb testing.TB, images []string, opts ...SandboxConfi
 	opts = append(opts, WithSandboxLabels(map[string]string{
 		"sandbox-platform": "linux/amd64",
 	}))
-	pullRequiredImagesWithOptions(tb, images, opts...)
+	// Set RuntimeHandler on ImageSpec so containerd CRI picks the LCOW
+	// runtime's configured snapshotter (windows-lcow) and platform
+	// (linux/amd64) rather than the default windows/amd64. The sandbox-
+	// platform label alone is not honored by modern containerd (≥2.0).
+	pullRequiredImagesWithRuntime(tb, images, lcowRuntimeHandlerForTest(tb), opts...)
 }
 
 func pullRequiredImagesWithOptions(tb testing.TB, images []string, opts ...SandboxConfigOpt) {
+	tb.Helper()
+	pullRequiredImagesWithRuntime(tb, images, "", opts...)
+}
+
+// pullRequiredImagesWithRuntime pulls each image with the given runtime
+// handler set on the CRI ImageSpec. Empty runtimeHandler means use the
+// containerd default. Tests pulling LCOW images should pass the LCOW handler
+// so containerd selects the windows-lcow snapshotter and linux/amd64 platform.
+func pullRequiredImagesWithRuntime(tb testing.TB, images []string, runtimeHandler string, opts ...SandboxConfigOpt) {
 	tb.Helper()
 	if len(images) < 1 {
 		return
@@ -228,7 +253,8 @@ func pullRequiredImagesWithOptions(tb testing.TB, images []string, opts ...Sandb
 	for _, image := range images {
 		_, err := client.PullImage(ctx, &runtime.PullImageRequest{
 			Image: &runtime.ImageSpec{
-				Image: image,
+				Image:          image,
+				RuntimeHandler: runtimeHandler,
 			},
 			SandboxConfig: sb,
 		})

test/cri-containerd/unmap_vpmem_test.go

@@ -28,6 +28,7 @@ const (
 // leave it intact and recycle, the bug no longer surfaces.
 func Test_Force_LayerUnmap_Not_In_Order(t *testing.T) {
 	requireFeatures(t, featureLCOW)
+	requireV1Only(t)
 
 	pullRequiredLCOWImages(t, []string{imageLcowK8sPause, ubuntu2004LargeLayers, ubuntu2204LargeLayers})
 

test/functional/helpers_v2_test.go

@@ -0,0 +1,25 @@
+//go:build windows && functional
+
+package functional
+
+import "testing"
+
+// isLCOWV2 reports whether the LCOWV2 feature flag is set on the current test
+// invocation. Callers should prefer the higher-level helper requireV1Only so
+// the V2 selection logic stays in one place.
+func isLCOWV2() bool {
+	return flagFeatures.IsSet(featureLCOWV2)
+}
+
+// requireV1Only skips the test when the LCOWV2 feature flag is set. Use this
+// for tests that depend on V1-only features such as VPMEM, VHD/initrd boot
+// modes, KernelDirect, or other UVM knobs not exposed in the v2 builder.
+//
+// Mirrors the pattern established in the azcri repo and in the CRI test suite
+// in test/cri-containerd/.
+func requireV1Only(tb testing.TB) {
+	tb.Helper()
+	if isLCOWV2() {
+		tb.Skip("test requires V1 shim features (VPMEM/VHD/initrd/KernelDirect) not exposed in V2")
+	}
+}