[docker in docker] Adds retries for docker daemon startup and cgroup nesting (#669)
* adds retry logic * adds stress test workflow * Adds cgroup retries * update workflow * move sleep cmd * update test * cleanup
This commit is contained in:
parent
a08434f2b1
commit
0cf690ee14
6 changed files with 139 additions and 31 deletions
37
.github/workflows/docker-in-docker-stress-test.yaml
vendored
Normal file
37
.github/workflows/docker-in-docker-stress-test.yaml
vendored
Normal file
|
@ -0,0 +1,37 @@
|
|||
name: "Stress test - Docker in Docker"
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'src/docker-in-docker/**'
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
test:
|
||||
strategy:
|
||||
matrix:
|
||||
test-pass: [ 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50 ]
|
||||
fail-fast: false
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: "Install latest devcontainer CLI"
|
||||
run: npm install -g @devcontainers/cli
|
||||
|
||||
- name: "Generating tests for 'docker-in-docker' which validates if docker daemon is running"
|
||||
run: devcontainer features test --skip-scenarios -f docker-in-docker -i mcr.microsoft.com/devcontainers/base:ubuntu .
|
||||
|
||||
test-onCreate:
|
||||
strategy:
|
||||
matrix:
|
||||
test-pass: [ 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50 ]
|
||||
fail-fast: false
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: "Install latest devcontainer CLI"
|
||||
run: npm install -g @devcontainers/cli
|
||||
|
||||
- name: "Generating tests for 'docker-in-docker' which validates if docker daemon is available within 'onCreateCommand'"
|
||||
run: devcontainer features test -f docker-in-docker --skip-autogenerated --filter "docker_with_on_create_command"
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"id": "docker-in-docker",
|
||||
"version": "2.3.1",
|
||||
"version": "2.4.0",
|
||||
"name": "Docker (Docker-in-Docker)",
|
||||
"documentationURL": "https://github.com/devcontainers/features/tree/main/src/docker-in-docker",
|
||||
"description": "Create child containers *inside* a container, independent from the host's docker instance. Installs Docker extension in the container along with needed CLIs.",
|
||||
|
|
|
@ -367,7 +367,7 @@ dockerd_start="AZURE_DNS_AUTO_DETECTION=${AZURE_DNS_AUTO_DETECTION} DOCKER_DEFAU
|
|||
find /run /var/run -iname 'docker*.pid' -delete || :
|
||||
find /run /var/run -iname 'container*.pid' -delete || :
|
||||
|
||||
## Dind wrapper script from docker team, adapted to a function
|
||||
# -- Start: dind wrapper script --
|
||||
# Maintained: https://github.com/moby/moby/blob/master/hack/dind
|
||||
|
||||
export container=docker
|
||||
|
@ -384,31 +384,52 @@ dockerd_start="AZURE_DNS_AUTO_DETECTION=${AZURE_DNS_AUTO_DETECTION} DOCKER_DEFAU
|
|||
mount -t tmpfs none /tmp
|
||||
fi
|
||||
|
||||
# cgroup v2: enable nesting
|
||||
if [ -f /sys/fs/cgroup/cgroup.controllers ]; then
|
||||
# move the processes from the root group to the /init group,
|
||||
# otherwise writing subtree_control fails with EBUSY.
|
||||
# An error during moving non-existent process (i.e., "cat") is ignored.
|
||||
mkdir -p /sys/fs/cgroup/init
|
||||
xargs -rn1 < /sys/fs/cgroup/cgroup.procs > /sys/fs/cgroup/init/cgroup.procs || :
|
||||
# enable controllers
|
||||
sed -e 's/ / +/g' -e 's/^/+/' < /sys/fs/cgroup/cgroup.controllers \
|
||||
> /sys/fs/cgroup/cgroup.subtree_control
|
||||
fi
|
||||
## Dind wrapper over.
|
||||
set_cgroup_nesting()
|
||||
{
|
||||
# cgroup v2: enable nesting
|
||||
if [ -f /sys/fs/cgroup/cgroup.controllers ]; then
|
||||
# move the processes from the root group to the /init group,
|
||||
# otherwise writing subtree_control fails with EBUSY.
|
||||
# An error during moving non-existent process (i.e., "cat") is ignored.
|
||||
mkdir -p /sys/fs/cgroup/init
|
||||
xargs -rn1 < /sys/fs/cgroup/cgroup.procs > /sys/fs/cgroup/init/cgroup.procs || :
|
||||
# enable controllers
|
||||
sed -e 's/ / +/g' -e 's/^/+/' < /sys/fs/cgroup/cgroup.controllers \
|
||||
> /sys/fs/cgroup/cgroup.subtree_control
|
||||
fi
|
||||
}
|
||||
|
||||
# Set cgroup nesting, retrying if necessary
|
||||
retry_cgroup_nesting=0
|
||||
|
||||
until [ "${retry_cgroup_nesting}" -eq "5" ];
|
||||
do
|
||||
set +e
|
||||
set_cgroup_nesting
|
||||
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "(*) cgroup v2: Failed to enable nesting, retrying..."
|
||||
else
|
||||
break
|
||||
fi
|
||||
|
||||
retry_cgroup_nesting=`expr $retry_cgroup_nesting + 1`
|
||||
set -e
|
||||
done
|
||||
|
||||
# -- End: dind wrapper script --
|
||||
|
||||
# Handle DNS
|
||||
set +e
|
||||
cat /etc/resolv.conf | grep -i 'internal.cloudapp.net'
|
||||
if [ $? -eq 0 ] && [ "${AZURE_DNS_AUTO_DETECTION}" = "true" ]
|
||||
then
|
||||
echo "Setting dockerd Azure DNS."
|
||||
CUSTOMDNS="--dns 168.63.129.16"
|
||||
else
|
||||
echo "Not setting dockerd DNS manually."
|
||||
CUSTOMDNS=""
|
||||
fi
|
||||
|
||||
cat /etc/resolv.conf | grep -i 'internal.cloudapp.net' > /dev/null 2>&1
|
||||
if [ $? -eq 0 ] && [ "${AZURE_DNS_AUTO_DETECTION}" = "true" ]
|
||||
then
|
||||
echo "Setting dockerd Azure DNS."
|
||||
CUSTOMDNS="--dns 168.63.129.16"
|
||||
else
|
||||
echo "Not setting dockerd DNS manually."
|
||||
CUSTOMDNS=""
|
||||
fi
|
||||
set -e
|
||||
|
||||
if [ -z "$DOCKER_DEFAULT_ADDRESS_POOL" ]
|
||||
|
@ -423,12 +444,35 @@ dockerd_start="AZURE_DNS_AUTO_DETECTION=${AZURE_DNS_AUTO_DETECTION} DOCKER_DEFAU
|
|||
INNEREOF
|
||||
)"
|
||||
|
||||
# Start using sudo if not invoked as root
|
||||
if [ "$(id -u)" -ne 0 ]; then
|
||||
sudo /bin/sh -c "${dockerd_start}"
|
||||
else
|
||||
eval "${dockerd_start}"
|
||||
fi
|
||||
retry_docker_start_count=0
|
||||
docker_ok="false"
|
||||
|
||||
until [ "${docker_ok}" = "true" ] || [ "${retry_docker_start_count}" -eq "5" ];
|
||||
do
|
||||
# Start using sudo if not invoked as root
|
||||
if [ "$(id -u)" -ne 0 ]; then
|
||||
sudo /bin/sh -c "${dockerd_start}"
|
||||
else
|
||||
eval "${dockerd_start}"
|
||||
fi
|
||||
|
||||
retry_count=0
|
||||
until [ "${docker_ok}" = "true" ] || [ "${retry_count}" -eq "5" ];
|
||||
do
|
||||
sleep 1s
|
||||
set +e
|
||||
docker info > /dev/null 2>&1 && docker_ok="true"
|
||||
set -e
|
||||
|
||||
retry_count=`expr $retry_count + 1`
|
||||
done
|
||||
|
||||
if [ "${docker_ok}" != "true" ]; then
|
||||
echo "(*) Failed to start docker, retrying..."
|
||||
fi
|
||||
|
||||
retry_docker_start_count=`expr $retry_docker_start_count + 1`
|
||||
done
|
||||
|
||||
# Execute whatever commands were passed in (if any). This allows us
|
||||
# to set this script to ENTRYPOINT while still executing the default CMD.
|
||||
|
|
18
test/docker-in-docker/docker_with_on_create_command.sh
Executable file
18
test/docker-in-docker/docker_with_on_create_command.sh
Executable file
|
@ -0,0 +1,18 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
# Optional: Import test library
|
||||
source dev-container-features-test-lib
|
||||
|
||||
# Feature specific tests
|
||||
check "version" docker --version
|
||||
check "docker-ps" bash -c "docker ps"
|
||||
|
||||
check "docker-init-exists" bash -c "ls /usr/local/share/docker-init.sh"
|
||||
check "log-exists" bash -c "ls /tmp/dockerd.log"
|
||||
check "log-for-completion" bash -c "cat /tmp/dockerd.log | grep 'Daemon has completed initialization'"
|
||||
check "log-contents" bash -c "cat /tmp/dockerd.log | grep 'API listen on /var/run/docker.sock'"
|
||||
|
||||
# Report result
|
||||
reportResults
|
|
@ -67,5 +67,14 @@
|
|||
}
|
||||
},
|
||||
"remoteUser": "node"
|
||||
},
|
||||
// DO NOT REMOVE: This scenario is used by the docker-in-docker-stress-test workflow
|
||||
"docker_with_on_create_command": {
|
||||
"image": "mcr.microsoft.com/devcontainers/base:debian",
|
||||
"features": {
|
||||
"docker-in-docker": {}
|
||||
},
|
||||
"remoteUser": "vscode",
|
||||
"onCreateCommand": "docker ps && sleep 5s && docker ps"
|
||||
}
|
||||
}
|
||||
|
|
|
@ -8,10 +8,10 @@ source dev-container-features-test-lib
|
|||
# Feature specific tests
|
||||
check "version" docker --version
|
||||
check "docker-init-exists" bash -c "ls /usr/local/share/docker-init.sh"
|
||||
check "docker-ps" bash -c "docker ps"
|
||||
check "log-exists" bash -c "ls /tmp/dockerd.log"
|
||||
check "log-for-completion" bash -c "cat /tmp/dockerd.log | grep 'Daemon has completed initialization'"
|
||||
check "log-contents" bash -c "cat /tmp/dockerd.log | grep 'API listen on /var/run/docker.sock'"
|
||||
check "docker-ps" bash -c "docker ps"
|
||||
|
||||
# Report result
|
||||
reportResults
|
Loading…
Reference in a new issue