From 39e43a7a74ce678b7ddcdc514fc48018ed525084 Mon Sep 17 00:00:00 2001 From: Erik Flodin Date: Tue, 29 Dec 2020 13:57:51 +0100 Subject: [PATCH 1/5] Remove the requirement on docker-compose for running tests Besides simplifying the setup it also has the nice side-effect of being able to override the docker image on the command line: $ make test IMAGE=foobar --- .github/CONTRIBUTING.md | 2 +- Makefile | 8 ++------ docker-compose.yml | 7 ------- 3 files changed, 3 insertions(+), 14 deletions(-) delete mode 100644 docker-compose.yml diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 23d1d9a..392a24d 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -207,7 +207,7 @@ these principles when making changes. ``` 4. Verify you can run the test harness. _(This will require dependencies: - `make`, `docker`, and `docker-compose`)_. + `make` and `docker`)_. ```text $ make test diff --git a/Makefile b/Makefile index 9154355..1a5cfc4 100644 --- a/Makefile +++ b/Makefile @@ -93,12 +93,8 @@ test: cd /yadm && \ py.test -v $(testargs); \ else \ - if command -v "docker-compose" > /dev/null 2>&1; then \ - docker-compose run --rm testbed make test testargs="$(testargs)"; \ - else \ - echo "Sorry, this make test requires docker-compose to be installed."; \ - false; \ - fi \ + $(MAKE) -s require-docker && \ + docker run --rm -it -v "$(CURDIR):/yadm:ro" $(IMAGE) make test testargs="$(testargs)"; \ fi .PHONY: testhost diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 2247667..0000000 --- a/docker-compose.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- -version: '3' -services: - testbed: - volumes: - - .:/yadm:ro - image: yadm/testbed:2020-12-21 From 6df2a5df746fd2d3445d4f696442387cb7932aa5 Mon Sep 17 00:00:00 2001 From: Erik Flodin Date: Tue, 29 Dec 2020 14:42:58 +0100 Subject: [PATCH 2/5] Shrink docker image Don't install apt recommendations and use the ADD directive instead of running curl in the image. Saves ~380MB in docker image size. --- test/Dockerfile | 77 +++++++++++++++++++++++++++---------------------- 1 file changed, 42 insertions(+), 35 deletions(-) diff --git a/test/Dockerfile b/test/Dockerfile index 7a1e684..fd46452 100644 --- a/test/Dockerfile +++ b/test/Dockerfile @@ -1,12 +1,29 @@ FROM ubuntu:18.04 MAINTAINER Tim Byrne -# No input during build -ENV DEBIAN_FRONTEND noninteractive +# Shellcheck and esh versions +ARG SC_VER=0.7.1 +ARG ESH_VER=0.3.0 + +# Install prerequisites and configure UTF-8 locale +RUN \ + echo "en_US.UTF-8 UTF-8" > /etc/locale.gen \ + && apt-get update \ + && DEBIAN_FRONTEND=noninteractive \ + apt-get install -y --no-install-recommends \ + expect \ + git \ + gnupg \ + locales \ + lsb-release \ + make \ + man \ + python3-pip \ + vim-tiny \ + xz-utils \ + && rm -rf /var/lib/apt/lists/* \ + && update-locale LANG='en_US.UTF-8' LANGUAGE='en_US:en' LC_ALL='en_US.UTF-8' -# UTF8 locale -RUN apt-get update && apt-get install -y locales -RUN locale-gen en_US.UTF-8 ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' LC_ALL='en_US.UTF-8' # Convenience settings for the testbed's root account @@ -15,43 +32,33 @@ RUN echo 'set -o vi' >> /root/.bashrc # Create a flag to identify when running inside the yadm testbed RUN touch /.yadmtestbed -# Install prerequisites -RUN \ - apt-get update && \ - apt-get install -y \ - curl \ - expect \ - git \ - gnupg \ - lsb-release \ - make \ - man \ - python3-pip \ - vim \ - && rm -rf /var/lib/apt/lists/*; - -ARG SC_VER=0.7.1 -RUN \ - curl -fLo - \ - https://github.com/koalaman/shellcheck/releases/download/v$SC_VER/shellcheck-v$SC_VER.linux.x86_64.tar.xz | \ - tar -xJv && \ - mv shellcheck-v$SC_VER/shellcheck /usr/bin && \ - rm -rf shellcheck-v$SC_VER +# Install shellcheck +ADD https://github.com/koalaman/shellcheck/releases/download/v$SC_VER/shellcheck-v$SC_VER.linux.x86_64.tar.xz /opt +RUN cd /opt \ + && tar xf shellcheck-v$SC_VER.linux.x86_64.tar.xz \ + && rm -f shellcheck-v$SC_VER.linux.x86_64.tar.xz \ + && ln -s /opt/shellcheck-v$SC_VER/shellcheck /usr/local/bin +# Upgrade pip3 and install requirements COPY test/requirements.txt /tmp/requirements.txt -RUN pip3 install --upgrade pip setuptools -RUN pip3 install --upgrade -r /tmp/requirements.txt +RUN python3 -m pip install --upgrade pip setuptools \ + && python3 -m pip install --upgrade -r /tmp/requirements.txt \ + && rm -f /tmp/requirements -RUN \ - curl https://raw.githubusercontent.com/jirutka/esh/v0.3.0/esh > /usr/local/bin/esh && \ - chmod +x /usr/local/bin/esh +# Install esh +ADD https://raw.githubusercontent.com/jirutka/esh/v$ESH_VER/esh /usr/local/bin +RUN chmod +x /usr/local/bin/esh + +# Create workdir and dummy Makefile to be used if no /yadm volume is mounted +RUN mkdir /yadm \ + && echo "test:" > /yadm/Makefile \ + && echo "\t@echo 'The yadm project must be mounted at /yadm'" >> /yadm/Makefile \ + && echo "\t@echo 'Try using a docker parameter like -v \"\$\$PWD:/yadm:ro\"'" >> /yadm/Makefile \ + && echo "\t@false" >> /yadm/Makefile # /yadm will be the work directory for all tests # docker commands should mount the local yadm project as /yadm WORKDIR /yadm -# Create a Makefile to be used if no /yadm volume is mounted -RUN echo "test:\n\t@echo 'The yadm project must be mounted at /yadm'\n\t@echo 'Try using a docker parameter like -v \"\$\$PWD:/yadm:ro\"'\n\t@false" > /yadm/Makefile - # By default, run all tests defined CMD make test From 6fc510f473fdf0428971e35e11e2987cce93e1fc Mon Sep 17 00:00:00 2001 From: Erik Flodin Date: Tue, 29 Dec 2020 19:49:41 +0100 Subject: [PATCH 3/5] Include released versions of yadm in docker images Make it easier to test upgrades (#276). --- test/Dockerfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/Dockerfile b/test/Dockerfile index fd46452..8b5d4df 100644 --- a/test/Dockerfile +++ b/test/Dockerfile @@ -56,6 +56,11 @@ RUN mkdir /yadm \ && echo "\t@echo 'Try using a docker parameter like -v \"\$\$PWD:/yadm:ro\"'" >> /yadm/Makefile \ && echo "\t@false" >> /yadm/Makefile +# Include released versions of yadm to test upgrades +ADD https://raw.githubusercontent.com/TheLocehiliosan/yadm/1.12.0/yadm /usr/local/bin/yadm-1.12.0 +ADD https://raw.githubusercontent.com/TheLocehiliosan/yadm/2.5.0/yadm /usr/local/bin/yadm-2.5.0 +RUN chmod +x /usr/local/bin/yadm-* + # /yadm will be the work directory for all tests # docker commands should mount the local yadm project as /yadm WORKDIR /yadm From ec3956c560744ecf976300a8701e974f84512668 Mon Sep 17 00:00:00 2001 From: Erik Flodin Date: Tue, 29 Dec 2020 19:51:55 +0100 Subject: [PATCH 4/5] Configure git email and name in docker to make it easier to use yadm in the container. Set it in the system config (i.e. /etc/gitconfig) to avoid getting a clash if one wants to clone an existing yadm repo that contains a .gitconfig. --- test/Dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/Dockerfile b/test/Dockerfile index 8b5d4df..3e5a299 100644 --- a/test/Dockerfile +++ b/test/Dockerfile @@ -61,6 +61,10 @@ ADD https://raw.githubusercontent.com/TheLocehiliosan/yadm/1.12.0/yadm /usr/loca ADD https://raw.githubusercontent.com/TheLocehiliosan/yadm/2.5.0/yadm /usr/local/bin/yadm-2.5.0 RUN chmod +x /usr/local/bin/yadm-* +# Configure git to make it easier to test yadm manually +RUN git config --system user.email "test@yadm.io" \ + && git config --system user.name "Yadm Test" + # /yadm will be the work directory for all tests # docker commands should mount the local yadm project as /yadm WORKDIR /yadm From 48e7337ef14733679ab780741767a769088bdf67 Mon Sep 17 00:00:00 2001 From: Erik Flodin Date: Tue, 29 Dec 2020 21:51:57 +0100 Subject: [PATCH 5/5] Upgrade to latest pytest version (6.2.1) --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 27ace6c..30da6ae 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ envtpl flake8==3.8.4 j2cli pylint==2.6.0 -pytest==6.1.2 +pytest==6.2.1 yamllint==1.25.0