[docker in docker] Adds retries for docker daemon startup and cgroup nesting (#669)
* adds retry logic * adds stress test workflow * Adds cgroup retries * update workflow * move sleep cmd * update test * cleanup
This commit is contained in:
parent
a08434f2b1
commit
0cf690ee14
6 changed files with 139 additions and 31 deletions
37
.github/workflows/docker-in-docker-stress-test.yaml
vendored
Normal file
37
.github/workflows/docker-in-docker-stress-test.yaml
vendored
Normal file
|
@ -0,0 +1,37 @@
|
||||||
|
name: "Stress test - Docker in Docker"
|
||||||
|
on:
|
||||||
|
pull_request:
|
||||||
|
paths:
|
||||||
|
- 'src/docker-in-docker/**'
|
||||||
|
workflow_dispatch:
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
test:
|
||||||
|
strategy:
|
||||||
|
matrix:
|
||||||
|
test-pass: [ 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50 ]
|
||||||
|
fail-fast: false
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v3
|
||||||
|
|
||||||
|
- name: "Install latest devcontainer CLI"
|
||||||
|
run: npm install -g @devcontainers/cli
|
||||||
|
|
||||||
|
- name: "Generating tests for 'docker-in-docker' which validates if docker daemon is running"
|
||||||
|
run: devcontainer features test --skip-scenarios -f docker-in-docker -i mcr.microsoft.com/devcontainers/base:ubuntu .
|
||||||
|
|
||||||
|
test-onCreate:
|
||||||
|
strategy:
|
||||||
|
matrix:
|
||||||
|
test-pass: [ 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50 ]
|
||||||
|
fail-fast: false
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v3
|
||||||
|
|
||||||
|
- name: "Install latest devcontainer CLI"
|
||||||
|
run: npm install -g @devcontainers/cli
|
||||||
|
|
||||||
|
- name: "Generating tests for 'docker-in-docker' which validates if docker daemon is available within 'onCreateCommand'"
|
||||||
|
run: devcontainer features test -f docker-in-docker --skip-autogenerated --filter "docker_with_on_create_command"
|
|
@ -1,6 +1,6 @@
|
||||||
{
|
{
|
||||||
"id": "docker-in-docker",
|
"id": "docker-in-docker",
|
||||||
"version": "2.3.1",
|
"version": "2.4.0",
|
||||||
"name": "Docker (Docker-in-Docker)",
|
"name": "Docker (Docker-in-Docker)",
|
||||||
"documentationURL": "https://github.com/devcontainers/features/tree/main/src/docker-in-docker",
|
"documentationURL": "https://github.com/devcontainers/features/tree/main/src/docker-in-docker",
|
||||||
"description": "Create child containers *inside* a container, independent from the host's docker instance. Installs Docker extension in the container along with needed CLIs.",
|
"description": "Create child containers *inside* a container, independent from the host's docker instance. Installs Docker extension in the container along with needed CLIs.",
|
||||||
|
|
|
@ -367,7 +367,7 @@ dockerd_start="AZURE_DNS_AUTO_DETECTION=${AZURE_DNS_AUTO_DETECTION} DOCKER_DEFAU
|
||||||
find /run /var/run -iname 'docker*.pid' -delete || :
|
find /run /var/run -iname 'docker*.pid' -delete || :
|
||||||
find /run /var/run -iname 'container*.pid' -delete || :
|
find /run /var/run -iname 'container*.pid' -delete || :
|
||||||
|
|
||||||
## Dind wrapper script from docker team, adapted to a function
|
# -- Start: dind wrapper script --
|
||||||
# Maintained: https://github.com/moby/moby/blob/master/hack/dind
|
# Maintained: https://github.com/moby/moby/blob/master/hack/dind
|
||||||
|
|
||||||
export container=docker
|
export container=docker
|
||||||
|
@ -384,6 +384,8 @@ dockerd_start="AZURE_DNS_AUTO_DETECTION=${AZURE_DNS_AUTO_DETECTION} DOCKER_DEFAU
|
||||||
mount -t tmpfs none /tmp
|
mount -t tmpfs none /tmp
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
set_cgroup_nesting()
|
||||||
|
{
|
||||||
# cgroup v2: enable nesting
|
# cgroup v2: enable nesting
|
||||||
if [ -f /sys/fs/cgroup/cgroup.controllers ]; then
|
if [ -f /sys/fs/cgroup/cgroup.controllers ]; then
|
||||||
# move the processes from the root group to the /init group,
|
# move the processes from the root group to the /init group,
|
||||||
|
@ -395,11 +397,31 @@ dockerd_start="AZURE_DNS_AUTO_DETECTION=${AZURE_DNS_AUTO_DETECTION} DOCKER_DEFAU
|
||||||
sed -e 's/ / +/g' -e 's/^/+/' < /sys/fs/cgroup/cgroup.controllers \
|
sed -e 's/ / +/g' -e 's/^/+/' < /sys/fs/cgroup/cgroup.controllers \
|
||||||
> /sys/fs/cgroup/cgroup.subtree_control
|
> /sys/fs/cgroup/cgroup.subtree_control
|
||||||
fi
|
fi
|
||||||
## Dind wrapper over.
|
}
|
||||||
|
|
||||||
|
# Set cgroup nesting, retrying if necessary
|
||||||
|
retry_cgroup_nesting=0
|
||||||
|
|
||||||
|
until [ "${retry_cgroup_nesting}" -eq "5" ];
|
||||||
|
do
|
||||||
|
set +e
|
||||||
|
set_cgroup_nesting
|
||||||
|
|
||||||
|
if [ $? -ne 0 ]; then
|
||||||
|
echo "(*) cgroup v2: Failed to enable nesting, retrying..."
|
||||||
|
else
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
|
||||||
|
retry_cgroup_nesting=`expr $retry_cgroup_nesting + 1`
|
||||||
|
set -e
|
||||||
|
done
|
||||||
|
|
||||||
|
# -- End: dind wrapper script --
|
||||||
|
|
||||||
# Handle DNS
|
# Handle DNS
|
||||||
set +e
|
set +e
|
||||||
cat /etc/resolv.conf | grep -i 'internal.cloudapp.net'
|
cat /etc/resolv.conf | grep -i 'internal.cloudapp.net' > /dev/null 2>&1
|
||||||
if [ $? -eq 0 ] && [ "${AZURE_DNS_AUTO_DETECTION}" = "true" ]
|
if [ $? -eq 0 ] && [ "${AZURE_DNS_AUTO_DETECTION}" = "true" ]
|
||||||
then
|
then
|
||||||
echo "Setting dockerd Azure DNS."
|
echo "Setting dockerd Azure DNS."
|
||||||
|
@ -408,7 +430,6 @@ dockerd_start="AZURE_DNS_AUTO_DETECTION=${AZURE_DNS_AUTO_DETECTION} DOCKER_DEFAU
|
||||||
echo "Not setting dockerd DNS manually."
|
echo "Not setting dockerd DNS manually."
|
||||||
CUSTOMDNS=""
|
CUSTOMDNS=""
|
||||||
fi
|
fi
|
||||||
|
|
||||||
set -e
|
set -e
|
||||||
|
|
||||||
if [ -z "$DOCKER_DEFAULT_ADDRESS_POOL" ]
|
if [ -z "$DOCKER_DEFAULT_ADDRESS_POOL" ]
|
||||||
|
@ -423,6 +444,11 @@ dockerd_start="AZURE_DNS_AUTO_DETECTION=${AZURE_DNS_AUTO_DETECTION} DOCKER_DEFAU
|
||||||
INNEREOF
|
INNEREOF
|
||||||
)"
|
)"
|
||||||
|
|
||||||
|
retry_docker_start_count=0
|
||||||
|
docker_ok="false"
|
||||||
|
|
||||||
|
until [ "${docker_ok}" = "true" ] || [ "${retry_docker_start_count}" -eq "5" ];
|
||||||
|
do
|
||||||
# Start using sudo if not invoked as root
|
# Start using sudo if not invoked as root
|
||||||
if [ "$(id -u)" -ne 0 ]; then
|
if [ "$(id -u)" -ne 0 ]; then
|
||||||
sudo /bin/sh -c "${dockerd_start}"
|
sudo /bin/sh -c "${dockerd_start}"
|
||||||
|
@ -430,6 +456,24 @@ else
|
||||||
eval "${dockerd_start}"
|
eval "${dockerd_start}"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
retry_count=0
|
||||||
|
until [ "${docker_ok}" = "true" ] || [ "${retry_count}" -eq "5" ];
|
||||||
|
do
|
||||||
|
sleep 1s
|
||||||
|
set +e
|
||||||
|
docker info > /dev/null 2>&1 && docker_ok="true"
|
||||||
|
set -e
|
||||||
|
|
||||||
|
retry_count=`expr $retry_count + 1`
|
||||||
|
done
|
||||||
|
|
||||||
|
if [ "${docker_ok}" != "true" ]; then
|
||||||
|
echo "(*) Failed to start docker, retrying..."
|
||||||
|
fi
|
||||||
|
|
||||||
|
retry_docker_start_count=`expr $retry_docker_start_count + 1`
|
||||||
|
done
|
||||||
|
|
||||||
# Execute whatever commands were passed in (if any). This allows us
|
# Execute whatever commands were passed in (if any). This allows us
|
||||||
# to set this script to ENTRYPOINT while still executing the default CMD.
|
# to set this script to ENTRYPOINT while still executing the default CMD.
|
||||||
exec "$@"
|
exec "$@"
|
||||||
|
|
18
test/docker-in-docker/docker_with_on_create_command.sh
Executable file
18
test/docker-in-docker/docker_with_on_create_command.sh
Executable file
|
@ -0,0 +1,18 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
set -e
|
||||||
|
|
||||||
|
# Optional: Import test library
|
||||||
|
source dev-container-features-test-lib
|
||||||
|
|
||||||
|
# Feature specific tests
|
||||||
|
check "version" docker --version
|
||||||
|
check "docker-ps" bash -c "docker ps"
|
||||||
|
|
||||||
|
check "docker-init-exists" bash -c "ls /usr/local/share/docker-init.sh"
|
||||||
|
check "log-exists" bash -c "ls /tmp/dockerd.log"
|
||||||
|
check "log-for-completion" bash -c "cat /tmp/dockerd.log | grep 'Daemon has completed initialization'"
|
||||||
|
check "log-contents" bash -c "cat /tmp/dockerd.log | grep 'API listen on /var/run/docker.sock'"
|
||||||
|
|
||||||
|
# Report result
|
||||||
|
reportResults
|
|
@ -67,5 +67,14 @@
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"remoteUser": "node"
|
"remoteUser": "node"
|
||||||
|
},
|
||||||
|
// DO NOT REMOVE: This scenario is used by the docker-in-docker-stress-test workflow
|
||||||
|
"docker_with_on_create_command": {
|
||||||
|
"image": "mcr.microsoft.com/devcontainers/base:debian",
|
||||||
|
"features": {
|
||||||
|
"docker-in-docker": {}
|
||||||
|
},
|
||||||
|
"remoteUser": "vscode",
|
||||||
|
"onCreateCommand": "docker ps && sleep 5s && docker ps"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -8,10 +8,10 @@ source dev-container-features-test-lib
|
||||||
# Feature specific tests
|
# Feature specific tests
|
||||||
check "version" docker --version
|
check "version" docker --version
|
||||||
check "docker-init-exists" bash -c "ls /usr/local/share/docker-init.sh"
|
check "docker-init-exists" bash -c "ls /usr/local/share/docker-init.sh"
|
||||||
|
check "docker-ps" bash -c "docker ps"
|
||||||
check "log-exists" bash -c "ls /tmp/dockerd.log"
|
check "log-exists" bash -c "ls /tmp/dockerd.log"
|
||||||
check "log-for-completion" bash -c "cat /tmp/dockerd.log | grep 'Daemon has completed initialization'"
|
check "log-for-completion" bash -c "cat /tmp/dockerd.log | grep 'Daemon has completed initialization'"
|
||||||
check "log-contents" bash -c "cat /tmp/dockerd.log | grep 'API listen on /var/run/docker.sock'"
|
check "log-contents" bash -c "cat /tmp/dockerd.log | grep 'API listen on /var/run/docker.sock'"
|
||||||
check "docker-ps" bash -c "docker ps"
|
|
||||||
|
|
||||||
# Report result
|
# Report result
|
||||||
reportResults
|
reportResults
|
Loading…
Reference in a new issue