From 638e8495c9c7c78d2f0c5120a90e0027bd1ac97f Mon Sep 17 00:00:00 2001 From: Chris Kuehl Date: Thu, 7 Oct 2021 11:19:38 -0700 Subject: [PATCH 1/5] Switch to GitHub Actions for CI --- .github/workflows/ci.yaml | 70 +++++++++++++++++++++++++++++++++++++++ .travis.yml | 29 ---------------- Dockerfile | 13 +++----- Makefile | 24 ++------------ ci/docker | 18 ---------- ci/docker-deb-test | 4 ++- ci/docker-python-test | 6 ++-- ci/docker-tox-test | 12 ------- pytest.ini | 2 +- tox.ini | 6 ++-- 10 files changed, 87 insertions(+), 97 deletions(-) create mode 100644 .github/workflows/ci.yaml delete mode 100644 .travis.yml delete mode 100644 ci/docker delete mode 100755 ci/docker-tox-test diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..83686b8 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,70 @@ +name: CI +on: push +jobs: + build-and-test: + runs-on: ubuntu-20.04 + strategy: + fail-fast: false + matrix: + include: + - arch: amd64 + docker_image: debian:buster + + - arch: arm64 + docker_image: arm64v8/debian:buster + + - arch: ppc64le + docker_image: ppc64le/debian:buster + + - arch: s390x + docker_image: s390x/debian:buster + + env: + BASE_IMAGE: ${{ matrix.docker_image }} + + steps: + - uses: actions/checkout@v2 + + - name: Set up QEMU + id: qemu + uses: docker/setup-qemu-action@v1 + if: ${{ matrix.arch != 'amd64' }} + with: + image: tonistiigi/binfmt:latest + + - name: Build Docker image + run: make docker-image + + - name: Run python tests + run: docker run --rm -v $(pwd):/mnt:rw dumb-init-build /mnt/ci/docker-python-test + + - name: Build Debian package + run: docker run --init --rm -v $(pwd):/mnt:rw dumb-init-build make -C /mnt builddeb + + - name: Test built Debian package + # XXX: This uses the clean base image (not the build one) to make + # sure it installs in a clean image without any hidden dependencies. + run: docker run --rm -v $(pwd):/mnt:rw ${{ matrix.docker_image }} /mnt/ci/docker-deb-test + + - name: Upload build artifacts + uses: actions/upload-artifact@v2 + with: + name: ${{ matrix.arch }} + path: dist + + # TODO: switch to pre-commit.ci + pre-commit: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.8' + + - name: Install tox + run: pip install tox + + - name: Run pre-commit + run: tox -e pre-commit diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 348f1c0..0000000 --- a/.travis.yml +++ /dev/null @@ -1,29 +0,0 @@ -language: c -services: - - docker - -matrix: - include: - - env: ITEST_TARGET=itest_focal - - env: ITEST_TARGET=itest_buster - - env: ITEST_TARGET=itest_tox - - arch: ppc64le - env: ITEST_TARGET=itest_buster - - arch: arm64 - env: ITEST_TARGET=itest_buster - - arch: s390x - env: ITEST_TARGET=itest_buster - allow_failures: - - arch: ppc64le - env: ITEST_TARGET=itest_buster - - arch: arm64 - env: ITEST_TARGET=itest_buster - - arch: s390x - env: ITEST_TARGET=itest_buster - - -script: - - make "$ITEST_TARGET" - -after_script: - - ci/artifact-upload diff --git a/Dockerfile b/Dockerfile index 73f1af8..c37efad 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,8 @@ -FROM debian:buster +ARG BASE_IMAGE=debian:buster +FROM $BASE_IMAGE LABEL maintainer="Chris Kuehl " -# The default mirrors are too flaky to run reliably in CI. -RUN sed -E \ - '/security\.debian/! s@http://[^/]+/@http://mirrors.kernel.org/@' \ - -i /etc/apt/sources.list - # Install the bare minimum dependencies necessary for working with Debian # packages. Build dependencies should be added under "Build-Depends" inside # debian/control instead. @@ -17,6 +13,9 @@ RUN : \ devscripts \ equivs \ lintian \ + python3-distutils \ + python3-setuptools \ + python3-pip \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* WORKDIR /tmp/mnt @@ -27,5 +26,3 @@ RUN : \ && mk-build-deps --install --tool 'apt-get -y --no-install-recommends' /control \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* - -ENTRYPOINT make builddeb diff --git a/Makefile b/Makefile index 78e8a35..f98c97d 100644 --- a/Makefile +++ b/Makefile @@ -54,11 +54,11 @@ builddeb: .PHONY: builddeb-docker builddeb-docker: docker-image mkdir -p dist - docker run --user $$(id -u):$$(id -g) -v $(PWD):/tmp/mnt dumb-init-build + docker run --init --user $$(id -u):$$(id -g) -v $(PWD):/tmp/mnt dumb-init-build make builddeb .PHONY: docker-image docker-image: - docker build -t dumb-init-build . + docker build $(if $(BASE_IMAGE),--build-arg BASE_IMAGE=$(BASE_IMAGE)) -t dumb-init-build . .PHONY: test test: @@ -68,23 +68,3 @@ test: .PHONY: install-hooks install-hooks: tox -e pre-commit -- install -f --install-hooks - -ITEST_TARGETS = itest_focal itest_buster - -.PHONY: itest $(ITEST_TARGETS) -itest: $(ITEST_TARGETS) - -itest_focal: _itest-ubuntu-focal -itest_buster: _itest-debian-buster - -itest_tox: - $(DOCKER_RUN_TEST) debian:buster /mnt/ci/docker-tox-test - -_itest-%: _itest_deb-% _itest_python-% - @true - -_itest_python-%: - $(DOCKER_RUN_TEST) $(shell sed 's/-/:/' <<< "$*") /mnt/ci/docker-python-test - -_itest_deb-%: builddeb-docker - $(DOCKER_RUN_TEST) $(shell sed 's/-/:/' <<< "$*") /mnt/ci/docker-deb-test diff --git a/ci/docker b/ci/docker deleted file mode 100644 index ced3ab1..0000000 --- a/ci/docker +++ /dev/null @@ -1,18 +0,0 @@ -# The default mirrors are too flaky to run reliably in CI. -sed -E \ - '/security\.debian/! s@http://[^/]+/@http://mirrors.kernel.org/@' \ - -i /etc/apt/sources.list - -apt-get update -apt-get install -y --no-install-recommends \ - build-essential \ - procps \ - python3 \ - python3-dev \ - python3-pip \ - python3-setuptools - -cp -r /mnt/ /test -cd /test - -# vim: ft=sh diff --git a/ci/docker-deb-test b/ci/docker-deb-test index 1e8f31f..cfe2d59 100755 --- a/ci/docker-deb-test +++ b/ci/docker-deb-test @@ -1,8 +1,10 @@ #!/bin/bash -eux set -o pipefail -. /mnt/ci/docker +apt-get update +apt-get -y --no-install-recommends install python3-pip procps +cd /mnt dpkg -i dist/*.deb pip3 install -r requirements-dev.txt pytest tests/ diff --git a/ci/docker-python-test b/ci/docker-python-test index 6daf48a..ecac947 100755 --- a/ci/docker-python-test +++ b/ci/docker-python-test @@ -1,12 +1,12 @@ #!/bin/bash -eux -set -o pipefail +set -euo pipefail -. /mnt/ci/docker +cd /mnt python3 setup.py clean python3 setup.py sdist pip3 install -vv dist/*.tar.gz pip3 install -r requirements-dev.txt -pytest tests/ +pytest-3 -vv tests/ exec dumb-init /mnt/tests/test-zombies diff --git a/ci/docker-tox-test b/ci/docker-tox-test deleted file mode 100755 index f515286..0000000 --- a/ci/docker-tox-test +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -eux -set -o pipefail - -. /mnt/ci/docker - -apt-get update -apt-get install -y --no-install-recommends \ - git \ - python3.7-dev \ - tox - -tox diff --git a/pytest.ini b/pytest.ini index 1ea6b80..13ce9a8 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,2 +1,2 @@ [pytest] -timeout = 5 +timeout = 20 diff --git a/tox.ini b/tox.ini index 1fed3e8..5a86cc6 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py37,gcov +envlist = py38,gcov [testenv] deps = -r{toxinidir}/requirements-dev.txt @@ -8,14 +8,14 @@ commands = [testenv:gcov] skip_install = True -basepython = /usr/bin/python3.7 +basepython = /usr/bin/python3.8 commands = {toxinidir}/ci/gcov-build {envbindir} {[testenv]commands} {toxinidir}/ci/gcov-report [testenv:pre-commit] -basepython = /usr/bin/python3.7 +basepython = /usr/bin/python3.8 commands = pre-commit {posargs:run --all-files} [flake8] From ba46e1be8137cba10b3cfe62e7002e1b3dfbf599 Mon Sep 17 00:00:00 2001 From: Chris Kuehl Date: Thu, 7 Oct 2021 11:21:16 -0700 Subject: [PATCH 2/5] Fix pre-commit --- setup.py | 2 -- testing/print_signals.py | 2 -- tests/cwd_test.py | 14 ++++++++++---- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/setup.py b/setup.py index 0ec2f18..0cb322d 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,3 @@ -from __future__ import print_function - import os.path import subprocess import tempfile diff --git a/testing/print_signals.py b/testing/print_signals.py index 4cc26b9..9fd7994 100755 --- a/testing/print_signals.py +++ b/testing/print_signals.py @@ -4,8 +4,6 @@ Since all signals are printed and otherwise ignored, you'll need to send SIGKILL (kill -9) to this process to actually end it. """ -from __future__ import print_function - import os import signal import sys diff --git a/tests/cwd_test.py b/tests/cwd_test.py index ff4ff80..d606bc8 100644 --- a/tests/cwd_test.py +++ b/tests/cwd_test.py @@ -1,9 +1,11 @@ import os import shutil -from subprocess import run, PIPE +from subprocess import PIPE +from subprocess import run import pytest + @pytest.mark.usefixtures('both_debug_modes', 'both_setsid_modes') def test_working_directories(): """The child process must start in the working directory in which @@ -14,9 +16,13 @@ def test_working_directories(): # predictable output - so we can't rely on dumb-init being found # in the "." directory. dumb_init = os.path.realpath(shutil.which('dumb-init')) - proc = run((dumb_init, - 'sh', '-c', 'readlink /proc/$PPID/cwd && readlink /proc/$$/cwd'), - cwd="/tmp", stdout=PIPE, stderr=PIPE) + proc = run( + ( + dumb_init, + 'sh', '-c', 'readlink /proc/$PPID/cwd && readlink /proc/$$/cwd', + ), + cwd='/tmp', stdout=PIPE, stderr=PIPE, + ) assert proc.returncode == 0 assert proc.stdout == b'/\n/tmp\n' From d2a1f7eae14dafb385f64ca66a2c84392796ca92 Mon Sep 17 00:00:00 2001 From: Chris Kuehl Date: Thu, 7 Oct 2021 11:21:21 -0700 Subject: [PATCH 3/5] Fix some tests that fail under GitHub CI --- testing/__init__.py | 11 ++++++++++- tests/cli_test.py | 28 +++++++++++++++++++++++++--- 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/testing/__init__.py b/testing/__init__.py index 665df65..80a4c66 100644 --- a/testing/__init__.py +++ b/testing/__init__.py @@ -52,7 +52,16 @@ def child_pids(pid): for p in LocalPath('/proc').listdir(): try: stat = open(p.join('stat').strpath).read() - m = re.match(r'^\d+ \(.+?\) [a-zA-Z] (\d+) ', stat) + m = re.match( + r'^\d+ \(.+?\) ' + # This field, state, is normally a single letter, but can be + # "0" if there are some unusual security settings that prevent + # reading the process state (happens under GitHub Actions with + # QEMU for some reason). + '[0a-zA-Z] ' + r'(\d+) ', + stat, + ) assert m, stat ppid = int(m.group(1)) if ppid == pid: diff --git a/tests/cli_test.py b/tests/cli_test.py index 27d7bf2..0142963 100644 --- a/tests/cli_test.py +++ b/tests/cli_test.py @@ -12,12 +12,34 @@ def current_version(): return open('VERSION').read().strip() +def normalize_stderr(stderr): + # dumb-init prints out argv[0] in its usage message. This should always be + # just "dumb-init" under regular test scenarios here since that is how we + # call it, but in CI the use of QEMU causes the argv[0] to be replaced with + # the full path. + # + # This is supposed to be avoidable by: + # 1) Setting the "P" flag in the binfmt register: + # https://en.wikipedia.org/wiki/Binfmt_misc#Registration + # This can be done by setting the QEMU_PRESERVE_PARENT env var when + # calling binfmt. + # + # 2) Setting the "QEMU_ARGV0" env var to empty string for *all* + # processes: + # https://bugs.launchpad.net/qemu/+bug/1835839 + # + # I can get it working properly in CI outside of Docker, but for some + # reason not during Docker builds. This doesn't affect the built executable + # so I decided to just punt on it. + return re.sub(rb'(^|(?<=\s))[a-z/.]+/dumb-init', b'dumb-init', stderr) + + @pytest.mark.usefixtures('both_debug_modes', 'both_setsid_modes') def test_no_arguments_prints_usage(): proc = Popen(('dumb-init'), stderr=PIPE) _, stderr = proc.communicate() assert proc.returncode != 0 - assert stderr == ( + assert normalize_stderr(stderr) == ( b'Usage: dumb-init [option] program [args]\n' b'Try dumb-init --help for full usage.\n' ) @@ -28,7 +50,7 @@ def test_exits_invalid_with_invalid_args(): proc = Popen(('dumb-init', '--yolo', '/bin/true'), stderr=PIPE) _, stderr = proc.communicate() assert proc.returncode == 1 - assert stderr in ( + assert normalize_stderr(stderr) in ( b"dumb-init: unrecognized option '--yolo'\n", # glibc b'dumb-init: unrecognized option: yolo\n', # musl ) @@ -43,7 +65,7 @@ def test_help_message(flag, current_version): proc = Popen(('dumb-init', flag), stderr=PIPE) _, stderr = proc.communicate() assert proc.returncode == 0 - assert stderr == ( + assert normalize_stderr(stderr) == ( b'dumb-init v' + current_version.encode('ascii') + b'\n' b'Usage: dumb-init [option] command [[arg] ...]\n' b'\n' From c7d5e17525a9cd73f6fc75232d17995c13cd470a Mon Sep 17 00:00:00 2001 From: Chris Kuehl Date: Thu, 7 Oct 2021 13:23:34 -0700 Subject: [PATCH 4/5] Remove Travis badge from the README --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 6c6dff7..87d6cf0 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,6 @@ dumb-init ======== -[![Travis CI](https://travis-ci.org/Yelp/dumb-init.svg?branch=master)](https://travis-ci.org/Yelp/dumb-init/) [![PyPI version](https://badge.fury.io/py/dumb-init.svg)](https://pypi.python.org/pypi/dumb-init) From 16ba926e92fce2cad85b702e7c25f5532785a6f8 Mon Sep 17 00:00:00 2001 From: Chris Kuehl Date: Thu, 7 Oct 2021 13:24:05 -0700 Subject: [PATCH 5/5] Remove pre-commit from CI, using pre-commit.ci instead --- .github/workflows/ci.yaml | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 83686b8..daafc8c 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -51,20 +51,3 @@ jobs: with: name: ${{ matrix.arch }} path: dist - - # TODO: switch to pre-commit.ci - pre-commit: - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v2 - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: '3.8' - - - name: Install tox - run: pip install tox - - - name: Run pre-commit - run: tox -e pre-commit