test updates to adopt https://github.com/devcontainers/cli/pull/124 (#114)

* test updates to adopt https://github.com/devcontainers/cli/pull/124

* remove accidental tgz check in

* typo in option

* typo in python install script on dotnet6/jammy branch

* no scenarios

* code review and test name

* Delete install_os_provided_python.sh

.github/workflows/test-all.yaml

@@ -38,12 +38,72 @@ jobs:
             "sshd",
             "terraform",
         ]
-        baseImage: [ "ubuntu:focal", "ubuntu:jammy", "debian:11", "mcr.microsoft.com/vscode/devcontainers/base:ubuntu", "mcr.microsoft.com/vscode/devcontainers/base:debian" ]
+        baseImage:
+          [
+            "ubuntu:focal",
+            "ubuntu:jammy",
+            "debian:11",
+            "mcr.microsoft.com/devcontainers/base:ubuntu",
+            "mcr.microsoft.com/devcontainers/base:debian",
+          ]
     steps:
       - uses: actions/checkout@v2
 
       - name: "Install latest devcontainer CLI"
-        run: npm install -g @devcontainers/cli@0.12.1
+        run: npm install -g @devcontainers/cli
 
-      - name: " Testing '${{ matrix.features }}' against '${{ matrix.baseImage }}'"
+      - name: "Generating tests for '${{ matrix.features }}' against '${{ matrix.baseImage }}'"
-        run: devcontainer features test -f ${{ matrix.features }} -i ${{ matrix.baseImage }} -c `pwd`
+        run: devcontainer features test  --skip-scenarios -f ${{ matrix.features }} -i ${{ matrix.baseImage }} .
+
+  test-scenarios:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    strategy:
+      matrix:
+        features: [
+            "anaconda",
+            "aws-cli",
+            "azure-cli",
+            "common-utils",
+            "desktop-lite",
+            "docker-from-docker",
+            "docker-in-docker",
+            "dotnet",
+            "git",
+            "git-lfs",
+            "github-cli",
+            "go",
+            "hugo",
+            "java",
+            "kubectl-helm-minikube",
+            "node",
+            "nvidia-cuda",
+            "oryx",
+            "php",
+            "powershell",
+            "python",
+            "ruby",
+            "rust",
+            "sshd",
+            "terraform",
+        ]
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: "Install latest devcontainer CLI"
+        run: npm install -g @devcontainers/cli
+
+      - name: "Testing '${{ matrix.features }}' scenarios"
+        run: devcontainer features test -f ${{ matrix.features }} --skip-autogenerated  .
+
+  test-global:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: "Install latest devcontainer CLI"
+        run: npm install -g @devcontainers/cli
+
+      - name: "Testing global scenarios"
+        run: devcontainer features test --global-scenarios-only .

.github/workflows/test-manual.yaml

@@ -3,7 +3,7 @@ on:
   workflow_dispatch:
     inputs:
       features:
-        description: "list of features to execute tests against"
+        description: "List of features to execute tests against"
         required: true
         default: "go dotnet"
       baseImage:
@@ -22,7 +22,7 @@ jobs:
       - uses: actions/checkout@v2
 
       - name: "Install latest devcontainer CLI"
-        run: npm install -g @devcontainers/cli@0.12.1
+        run: npm install -g @devcontainers/cli
 
       - name: "Testing '${{ github.event.inputs.features }}' against '${{ github.event.inputs.baseImage }}'"
-        run: devcontainer features test --features ${{ github.event.inputs.features }} --base-image ${{ github.event.inputs.baseImage }} --collection-folder `pwd` --log-level ${{ github.event.inputs.logLevel }}
+        run: devcontainer features test --features ${{ github.event.inputs.features }} --base-image ${{ github.event.inputs.baseImage }} --log-level ${{ github.event.inputs.logLevel }} .

.github/workflows/test-pr.yaml

@@ -1,4 +1,4 @@
-name: "PR -Test Updated Features"
+name: "PR - Test Updated Features"
 on:
   pull_request:
 
@@ -50,14 +50,30 @@ jobs:
             "ubuntu:focal",
             "ubuntu:jammy",
             "debian:11",
-            "mcr.microsoft.com/vscode/devcontainers/base:ubuntu",
+            "mcr.microsoft.com/devcontainers/base:ubuntu",
-            "mcr.microsoft.com/vscode/devcontainers/base:debian",
+            "mcr.microsoft.com/devcontainers/base:debian",
           ]
     steps:
       - uses: actions/checkout@v2
 
       - name: "Install latest devcontainer CLI"
-        run: npm install -g @devcontainers/cli@0.12.1
+        run: npm install -g @devcontainers/cli
 
-      - name: "Testing '${{ matrix.features }}' against '${{ matrix.baseImage }}'"
+      - name: "Generating tests for '${{ matrix.features }}' against '${{ matrix.baseImage }}'"
-        run: devcontainer features test -f ${{ matrix.features }} -i ${{ matrix.baseImage }} -c `pwd`
+        run: devcontainer features test  --skip-scenarios -f ${{ matrix.features }} -i ${{ matrix.baseImage }} .
+
+  test-scenarios:
+    needs: [detect-changes]
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    strategy:
+      matrix:
+        features: ${{ fromJSON(needs.detect-changes.outputs.features) }}
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: "Install latest devcontainer CLI"
+        run: npm install -g @devcontainers/cli
+
+      - name: "Testing '${{ matrix.features }}' scenarios"
+        run: devcontainer features test -f ${{ matrix.features }} --skip-autogenerated  .

.github/workflows/test-scenarios.yaml

@@ -1,19 +0,0 @@
-name: "Test Scenarios"
-on:
-  pull_request:
-  push:
-    branches:
-      - main
-  workflow_dispatch:
-
-jobs:
-  scenarios:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: "Install latest devcontainer CLI"
-        run: npm install -g @devcontainers/cli@0.12.1
-
-      - name: "Running predefined test scenarios"
-        run: devcontainer features test --scenarios `pwd`/test-scenarios -c `pwd`

test-scenarios/install_python_twice.sh

@@ -1,14 +0,0 @@
-#!/bin/bash
-
-set -e
-
-# Optional: Import test library
-source dev-container-features-test-lib
-
-check "version 3.9 installed"  ls -1 /usr/local/python/ | grep 3.9
-check "version 3.8 installed"  ls -1 /usr/local/python/ | grep 3.8
-
-check "3.9 alias to python on path" python  --version | grep 3.9
-
-# Report result
-reportResults

test-scenarios/scenarios.json

@@ -1,130 +0,0 @@
-{
-    "install_os_provided_python": {
-        "image": "mcr.microsoft.com/devcontainers/base:0-bullseye",
-        "features": {
-            "node": "latest",
-            "python": "os-provided"
-        }
-    },
-    "install_gradle_and_maven": {
-        "image": "ubuntu:focal",
-        "features": {
-            "java": {
-                "version": "latest",
-                "installGradle": true,
-                "installMaven": true
-            }
-        }
-    },
-    "install_additional_php": {
-        "image": "ubuntu:focal",
-        "features": {
-            "php": {
-                "version": "8.1.4",
-                "additionalVersions": "8.0.17,8.0.3"
-            }
-        }
-    },
-    "install_additional_java": {
-        "image": "ubuntu:focal",
-        "features": {
-            "java": {
-                "version": "17",
-                "additionalVersions": "11,8"
-            }
-        }
-    },
-    "install_jupyterlab": {
-        "image": "mcr.microsoft.com/devcontainers/base:focal",
-        "remoteUser": "vscode",
-        "features": {
-            "common-utils": {
-                "username": "vscode"
-            },
-            "python": {
-                "installJupyterlab": true,
-                "configureJupyterlabAllowOrigin": "*"
-            }
-        }
-    },
-    "install_additional_node": {
-        "image": "debian:11",
-        "features": {
-            "node": {
-                "version": "latest",
-                "additionalVersions": "v17.9.1,v14.19.3"
-            }
-        }
-    },
-    "install_additional_python": {
-        "image": "ubuntu:focal",
-        "features": {
-            "python": {
-                "version": "3.10.5",
-                "additionalVersions": "3.8,3.9.13"
-            }
-        }
-    },
-    "install_additional_ruby": {
-        "image": "ubuntu:focal",
-        "features": {
-            "ruby": {
-                "version": "3.1.2",
-                "additionalVersions": "2.5,3.0.4"
-            }
-        }
-    },
-    "install_additional_dotnet": {
-        "image": "ubuntu:focal",
-        "features": {
-            "dotnet": {
-                "version": "6.0.301",
-                "additionalVersions": "5.0,3.1.420"
-            }
-        }
-    },
-    "install_dotnet_5": {
-        "image": "ubuntu:focal",
-        "features": {
-            "dotnet": {
-                "version": "5"
-            }
-        }
-    },
-    "install_dotnet_3": {
-        "image": "ubuntu:focal",
-        "features": {
-            "dotnet": {
-                "version": "3"
-            }
-        }
-    },
-    "rust_at_pinned_version": {
-        "image": "ubuntu:focal",
-        "features": {
-            "rust": {
-                "version": "1.62.0"
-            }
-        }
-    },
-    "install_cudnn_nvxt": {
-        "image": "debian",
-        "features": {
-            "nvidia-cuda": {
-                "installCudnn": true,
-                "installNvtx": true
-            }
-        }
-    },
-    "install_cudnn_nvxt_version": {
-        "image": "debian",
-        "features": {
-            "nvidia-cuda": {
-                "installCudnn": true,
-                "installNvtx": true,
-                "cudaVersion": "11.5",
-                "cudnnVersion": "8.3.2.44"
-            }
-        }
-    }
-}

test/_global/all_the_clis.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -e
+
+# Optional: Import test library
+source dev-container-features-test-lib
+
+# Definition specific tests
+check "check for aws" aws --version
+check "check for gh" gh --version
+check "check for azure" az  --version
+
+# Report result
+reportResults

test/_global/node_java_rust.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -e
+
+# Optional: Import test library
+source dev-container-features-test-lib
+
+# Definition specific tests
+check "check for node" node --version
+check "check for java" java --version
+check "check for rust" rustc  --version
+
+# Report result
+reportResults

test/_global/scenarios.json

@@ -0,0 +1,18 @@
+{
+    "all_the_clis": {
+        "image": "ubuntu:focal",
+        "features": {
+            "aws-cli": {},
+            "azure-cli": {},
+            "github-cli": {}
+        }
+    },
+    "node_java_rust": {
+        "image": "ubuntu:focal",
+        "features": {
+            "node": {},
+            "java": {},
+            "rust": {}
+        }
+    }
+}

test/dotnet/scenarios.json

@@ -0,0 +1,27 @@
+{
+    "install_additional_dotnet": {
+        "image": "ubuntu:focal",
+        "features": {
+            "dotnet": {
+                "version": "6.0.301",
+                "additionalVersions": "5.0,3.1.420"
+            }
+        }
+    },
+    "install_dotnet_3": {
+        "image": "ubuntu:focal",
+        "features": {
+            "dotnet": {
+                "version": "3"
+            }
+        }
+    },
+    "install_dotnet_5": {
+        "image": "ubuntu:focal",
+        "features": {
+            "dotnet": {
+                "version": "5"
+            }
+        }
+    }
+}

test/java/scenarios.json

@@ -0,0 +1,21 @@
+{
+    "install_additional_java": {
+        "image": "ubuntu:focal",
+        "features": {
+            "java": {
+                "version": "17",
+                "additionalVersions": "11,8"
+            }
+        }
+    },
+    "install_gradle_and_maven": {
+        "image": "ubuntu:focal",
+        "features": {
+            "java": {
+                "version": "latest",
+                "installGradle": true,
+                "installMaven": true
+            }
+        }
+    }
+}

test/node/scenarios.json

@@ -0,0 +1,11 @@
+{
+    "install_additional_node": {
+        "image": "debian:11",
+        "features": {
+            "node": {
+                "version": "latest",
+                "additionalVersions": "v17.9.1,v14.19.3"
+            }
+        }
+    }
+}

test/nvidia-cuda/scenarios.json

@@ -0,0 +1,22 @@
+{
+    "install_cudnn_nvxt": {
+        "image": "debian",
+        "features": {
+            "nvidia-cuda": {
+                "installCudnn": true,
+                "installNvtx": true
+            }
+        }
+    },
+    "install_cudnn_nvxt_version": {
+        "image": "debian",
+        "features": {
+            "nvidia-cuda": {
+                "installCudnn": true,
+                "installNvtx": true,
+                "cudaVersion": "11.5",
+                "cudnnVersion": "8.3.2.44"
+            }
+        }
+    }
+}

test/php/scenarios.json

@@ -0,0 +1,11 @@
+{
+    "install_additional_php": {
+        "image": "ubuntu:focal",
+        "features": {
+            "php": {
+                "version": "8.1.4",
+                "additionalVersions": "8.0.17,8.0.3"
+            }
+        }
+    }
+}

test/python/scenarios.json

@@ -0,0 +1,31 @@
+{
+    "install_additional_python": {
+        "image": "ubuntu:focal",
+        "features": {
+            "python": {
+                "version": "3.10.5",
+                "additionalVersions": "3.8,3.9.13"
+            }
+        }
+    },
+    "install_jupyterlab": {
+        "image": "mcr.microsoft.com/vscode/devcontainers/base:focal",
+        "remoteUser": "vscode",
+        "features": {
+            "common-utils": {
+                "username": "vscode"
+            },
+            "python": {
+                "installJupyterlab": true,
+                "configureJupyterlabAllowOrigin": "*"
+            }
+        }
+    },
+    "install_os_provided_python": {
+        "image": "mcr.microsoft.com/devcontainers/base:0-bullseye",
+        "features": {
+            "node": "latest",
+            "python": "os-provided"
+        }
+    }
+}

test/ruby/scenarios.json

@@ -0,0 +1,11 @@
+{
+    "install_additional_ruby": {
+        "image": "ubuntu:focal",
+        "features": {
+            "ruby": {
+                "version": "3.1.2",
+                "additionalVersions": "2.5,3.0.4"
+            }
+        }
+    }
+}

test/rust/scenarios.json

@@ -0,0 +1,10 @@
+{
+    "rust_at_pinned_version": {
+        "image": "ubuntu:focal",
+        "features": {
+            "rust": {
+                "version": "1.62.0"
+            }
+        }
+    }
+}