[docker in docker] Adds retries for docker daemon startup and cgroup nesting (#669)

* adds retry logic

* adds stress test workflow

* Adds cgroup retries

* update workflow

* move sleep cmd

* update test

* cleanup

.github/workflows/docker-in-docker-stress-test.yaml

@@ -0,0 +1,37 @@
+name: "Stress test - Docker in Docker"
+on:
+  pull_request:
+    paths:
+      - 'src/docker-in-docker/**'
+  workflow_dispatch:
+
+jobs:
+  test:
+    strategy:
+      matrix:
+        test-pass: [ 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50 ]
+      fail-fast: false
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: "Install latest devcontainer CLI"
+        run: npm install -g @devcontainers/cli
+
+      - name: "Generating tests for 'docker-in-docker' which validates if docker daemon is running"
+        run: devcontainer features test --skip-scenarios -f docker-in-docker -i mcr.microsoft.com/devcontainers/base:ubuntu .
+
+  test-onCreate:
+    strategy:
+      matrix:
+        test-pass: [ 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50 ]
+      fail-fast: false
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: "Install latest devcontainer CLI"
+        run: npm install -g @devcontainers/cli
+
+      - name: "Generating tests for 'docker-in-docker' which validates if docker daemon is available within 'onCreateCommand'"
+        run: devcontainer features test -f docker-in-docker --skip-autogenerated --filter "docker_with_on_create_command"

src/docker-in-docker/devcontainer-feature.json

@@ -1,6 +1,6 @@
 {
     "id": "docker-in-docker",
-    "version": "2.3.1",
+    "version": "2.4.0",
     "name": "Docker (Docker-in-Docker)",
     "documentationURL": "https://github.com/devcontainers/features/tree/main/src/docker-in-docker",
     "description": "Create child containers *inside* a container, independent from the host's docker instance. Installs Docker extension in the container along with needed CLIs.",

src/docker-in-docker/install.sh

@@ -367,7 +367,7 @@ dockerd_start="AZURE_DNS_AUTO_DETECTION=${AZURE_DNS_AUTO_DETECTION} DOCKER_DEFAU
     find /run /var/run -iname 'docker*.pid' -delete || :
     find /run /var/run -iname 'container*.pid' -delete || :
 
-    ## Dind wrapper script from docker team, adapted to a function
+    # -- Start: dind wrapper script --
     # Maintained: https://github.com/moby/moby/blob/master/hack/dind
 
     export container=docker
@@ -384,6 +384,8 @@ dockerd_start="AZURE_DNS_AUTO_DETECTION=${AZURE_DNS_AUTO_DETECTION} DOCKER_DEFAU
         mount -t tmpfs none /tmp
     fi
 
+    set_cgroup_nesting()
+    {
         # cgroup v2: enable nesting
         if [ -f /sys/fs/cgroup/cgroup.controllers ]; then
             # move the processes from the root group to the /init group,
@@ -395,11 +397,31 @@ dockerd_start="AZURE_DNS_AUTO_DETECTION=${AZURE_DNS_AUTO_DETECTION} DOCKER_DEFAU
             sed -e 's/ / +/g' -e 's/^/+/' < /sys/fs/cgroup/cgroup.controllers \
                 > /sys/fs/cgroup/cgroup.subtree_control
         fi
-    ## Dind wrapper over.
+    }
+
+    # Set cgroup nesting, retrying if necessary
+    retry_cgroup_nesting=0
+
+    until [ "${retry_cgroup_nesting}" -eq "5" ];
+    do
+        set +e
+            set_cgroup_nesting
+
+            if [ $? -ne 0 ]; then
+                echo "(*) cgroup v2: Failed to enable nesting, retrying..."
+            else
+                break
+            fi
+
+            retry_cgroup_nesting=`expr $retry_cgroup_nesting + 1`
+        set -e
+    done 
+
+    # -- End: dind wrapper script --
 
     # Handle DNS
     set +e
-    cat /etc/resolv.conf | grep -i 'internal.cloudapp.net'
+        cat /etc/resolv.conf | grep -i 'internal.cloudapp.net' > /dev/null 2>&1
         if [ $? -eq 0 ] && [ "${AZURE_DNS_AUTO_DETECTION}" = "true" ]
         then
             echo "Setting dockerd Azure DNS."
@@ -408,7 +430,6 @@ dockerd_start="AZURE_DNS_AUTO_DETECTION=${AZURE_DNS_AUTO_DETECTION} DOCKER_DEFAU
             echo "Not setting dockerd DNS manually."
             CUSTOMDNS=""
         fi
-
     set -e
 
     if [ -z "$DOCKER_DEFAULT_ADDRESS_POOL" ]
@@ -423,12 +444,35 @@ dockerd_start="AZURE_DNS_AUTO_DETECTION=${AZURE_DNS_AUTO_DETECTION} DOCKER_DEFAU
 INNEREOF
 )"
 
-# Start using sudo if not invoked as root
+retry_docker_start_count=0
-if [ "$(id -u)" -ne 0 ]; then
+docker_ok="false"
+
+until [ "${docker_ok}" = "true"  ] || [ "${retry_docker_start_count}" -eq "5" ];
+do 
+    # Start using sudo if not invoked as root
+    if [ "$(id -u)" -ne 0 ]; then
         sudo /bin/sh -c "${dockerd_start}"
-else
+    else
         eval "${dockerd_start}"
-fi
+    fi
+
+    retry_count=0
+    until [ "${docker_ok}" = "true"  ] || [ "${retry_count}" -eq "5" ];
+    do
+        sleep 1s
+        set +e
+            docker info > /dev/null 2>&1 && docker_ok="true"
+        set -e
+
+        retry_count=`expr $retry_count + 1`
+    done
+    
+    if [ "${docker_ok}" != "true" ]; then
+        echo "(*) Failed to start docker, retrying..."
+    fi
+    
+    retry_docker_start_count=`expr $retry_docker_start_count + 1`
+done
 
 # Execute whatever commands were passed in (if any). This allows us
 # to set this script to ENTRYPOINT while still executing the default CMD.

test/docker-in-docker/docker_with_on_create_command.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -e
+
+# Optional: Import test library
+source dev-container-features-test-lib
+
+# Feature specific tests
+check "version" docker  --version
+check "docker-ps" bash -c "docker ps"
+
+check "docker-init-exists" bash -c "ls /usr/local/share/docker-init.sh"
+check "log-exists" bash -c "ls /tmp/dockerd.log"
+check "log-for-completion" bash -c "cat /tmp/dockerd.log | grep 'Daemon has completed initialization'"
+check "log-contents" bash -c "cat /tmp/dockerd.log | grep 'API listen on /var/run/docker.sock'"
+
+# Report result
+reportResults

test/docker-in-docker/scenarios.json

@@ -67,5 +67,14 @@
             }
         },
         "remoteUser": "node"
+    },
+    // DO NOT REMOVE: This scenario is used by the docker-in-docker-stress-test workflow
+    "docker_with_on_create_command": {
+        "image": "mcr.microsoft.com/devcontainers/base:debian",
+        "features": {
+            "docker-in-docker": {}
+        },
+        "remoteUser": "vscode",
+        "onCreateCommand": "docker ps && sleep 5s && docker ps"
     }
 }

test/docker-in-docker/test.sh

@@ -8,10 +8,10 @@ source dev-container-features-test-lib
 # Feature specific tests
 check "version" docker  --version
 check "docker-init-exists" bash -c "ls /usr/local/share/docker-init.sh"
+check "docker-ps" bash -c "docker ps"
 check "log-exists" bash -c "ls /tmp/dockerd.log"
 check "log-for-completion" bash -c "cat /tmp/dockerd.log | grep 'Daemon has completed initialization'"
 check "log-contents" bash -c "cat /tmp/dockerd.log | grep 'API listen on /var/run/docker.sock'"
-check "docker-ps" bash -c "docker ps"
 
 # Report result
 reportResults