From 7eec3ad972397ff217b456378c8f9365ea56f8e6 Mon Sep 17 00:00:00 2001 From: Marcello de Sales Date: Tue, 22 Sep 2020 19:12:06 -0300 Subject: [PATCH 1/4] :hammer: Dockerfile: add stages for specifics According to the Buildkit library of docker, we can make parallel builds easier with multi-stage Dockerfiles. https://docs.docker.com/develop/develop-images/build_enhancements/#to-enable-buildkit-builds Here's an example: $ cat Dockerfile2 FROM alpine AS dependencies RUN echo "Downloading dependencies" RUN sleep 15 && echo "dependencies done" FROM dependencies AS windows RUN echo "Compiling windows" RUN sleep 10 && echo "windows done" FROM dependencies AS linux RUN echo "Compiling linux" RUN sleep 10 && echo "linux done" $ DOCKER_BUILDKIT=1 docker build -t build-dependencies -f ./Dockerfile2 --target dependencies . [+] Building 0.1s (7/7) FINISHED => [internal] load build definition from Dockerfile2 0.0s => => transferring dockerfile: 38B 0.0s => [internal] load .dockerignore 0.0s => => transferring context: 2B 0.0s => [internal] load metadata for docker.io/library/alpine:latest 0.0s => [dependencies 1/3] FROM docker.io/library/alpine 0.0s => CACHED [dependencies 2/3] RUN echo "Downloading dependencies" 0.0s => CACHED [dependencies 3/3] RUN sleep 15 && echo "dependencies done" 0.0s => exporting to image 0.0s => => exporting layers 0.0s => => writing image sha256:b70fc38c238cddf7d88cb495634ce79cb49b0b9416018746e8523433f91a6015 0.0s => => naming to docker.io/library/build-dependencies 0.0s * Here, another feature is to use the build cache from the previous image built for dependencies. In addition, reusing the cache is shown as CACHED for the dependencies and it avoids building the entire docker image stages (when not using BUILDKIT=1) $ DOCKER_BUILDKIT=1 docker build -t windows-binary -f ./Dockerfile2 --target windows --cache-from=build-dependencies . [+] Building 0.0s (10/10) FINISHED => [internal] load .dockerignore 0.0s => => transferring context: 2B 0.0s => [internal] load build definition from Dockerfile2 0.0s => => transferring dockerfile: 38B 0.0s => [internal] load metadata for docker.io/library/alpine:latest 0.0s => importing cache manifest from build-dependencies 0.0s => [dependencies 1/3] FROM docker.io/library/alpine 0.0s => CACHED [dependencies 2/3] RUN echo "Downloading dependencies" 0.0s => CACHED [dependencies 3/3] RUN sleep 15 && echo "dependencies done" 0.0s => CACHED [windows 1/2] RUN echo "Compiling windows" 0.0s => CACHED [windows 2/2] RUN sleep 10 && echo "windows done" 0.0s => exporting to image 0.0s => => exporting layers 0.0s => => writing image sha256:cb2fdc9acfa878f50314d03e644f205729c8314c90d8323fb9203bbc16227275 0.0s => => naming to docker.io/library/windows-binary * Same here for the linux binaries, we can build and only the linux ones will be built with the help of the cache built previously. $ DOCKER_BUILDKIT=1 docker build -t linux-binary -f ./Dockerfile2 --target linux --cache-from=build-dependencies . [+] Building 0.1s (10/10) FINISHED => [internal] load build definition from Dockerfile2 0.0s => => transferring dockerfile: 38B 0.0s => [internal] load .dockerignore 0.0s => => transferring context: 2B 0.0s => [internal] load metadata for docker.io/library/alpine:latest 0.0s => importing cache manifest from build-dependencies 0.0s => [dependencies 1/3] FROM docker.io/library/alpine 0.0s => CACHED [dependencies 2/3] RUN echo "Downloading dependencies" 0.0s => CACHED [dependencies 3/3] RUN sleep 15 && echo "dependencies done" 0.0s => CACHED [linux 1/2] RUN echo "Compiling linux" 0.0s => CACHED [linux 2/2] RUN sleep 10 && echo "linux done" 0.0s => exporting to image 0.0s => => exporting layers 0.0s => => writing image sha256:35c85e91b47966432ae3d3c60bc455587bc46016eb528674b51b3f8c86a1e9da 0.0s => => naming to docker.io/library/linux-binary 0.0s Our builds are to run with the same level of separation. *** For our builds with BUILDKIT, we have the following => Dockerifle * dependencies: downloads all the OS and Golang specific dependencies $ DOCKER_BUILDKIT=1 BIN_VERSION=20.09.10 docker-compose build dependencies * compiler: compiles the go code with the dependencies layer $ DOCKER_BUILDKIT=1 BIN_VERSION=20.09.10 PLATFORMS=darwin docker-compose build binaries * runtime: specific for the linux runtime $ DOCKER_BUILDKIT=1 PLATFORMS=linux BIN_VERSION=20.09.10 docker-compose build runtime --- Dockerfile | 12 ++++++++---- docker-compose.yaml | 29 +++++++++++++++++++++++++++-- 2 files changed, 35 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 75bc7cd..dff3d8c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ #first stage - builder -FROM golang:1.13.0-stretch as builder +FROM golang:1.13.0-stretch AS dependencies WORKDIR /build @@ -18,6 +18,8 @@ COPY go.sum /build/go.sum ENV GO111MODULE=on RUN go mod download +FROM dependencies AS compiler + # Add the main COPY main.go /build/main.go @@ -33,11 +35,13 @@ COPY util /build/util ARG BIN_NAME ARG BIN_VERSION +ARG PLATFORMS # Cross-compile all versions ENV BIN_NAME ${BIN_NAME:-unknown} ENV BUILD_VERSION ${BIN_VERSION:-0.1.0} -ENV PLATFORMS "darwin linux windows" +ENV PLATFORMS ${PLATFORMS:-darwin linux windows} +RUN echo "Building for ${PLATFORMS}" ENV ARCHS "amd64" #ENV ARCHS "386 amd64" @@ -58,14 +62,14 @@ RUN export export FULL_NAME_GIT=$(git -C /build/.git remote -v | grep fetch | aw #RUN upx --lzma /build/${BIN_NAME}* # Build the main container (Linux Runtime) -FROM alpine:latest +FROM alpine:latest AS runtime WORKDIR /root/ ARG BIN_NAME ENV BIN_NAME ${BIN_NAME:-unknown} # Copy the linux amd64 binary, based on the arg (or else all files are copied) inspect with https://github.com/wagoodman/dive -COPY --from=builder /build/${BIN_NAME}* /usr/local/bin/ +COPY --from=compiler /build/${BIN_NAME}* /usr/local/bin/ # Move the bin to /usr/local/bin and make the entrypoint to point to it passing the params # https://stackoverflow.com/questions/33439230/how-to-write-commands-with-multiple-lines-in-dockerfile-while-preserving-the-new/33439625#33439625 diff --git a/docker-compose.yaml b/docker-compose.yaml index 42357d2..56101a8 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,11 +1,36 @@ -version: "3" +version: "3.8" services: - cli: + # Usually takes time to download, and can be reused for all platforms + dependencies: + image: marcellodesales/cloner-dependencies:${BIN_VERSION:-0.1.0} + build: + context: . + target: dependencies + + # Reuses the dependencies image as cache so we can parallelize binary builds + binaries: + image: marcellodesales/cloner-binaries-${PLATFORMS}:${BIN_VERSION:-0.1.0} + build: + context: . + args: + - BIN_NAME=cloner + - BIN_VERSION=${BIN_VERSION:-0.1.0} + - PLATFORMS=${PLATFORMS} + cache_from: + - marcellodesales/cloner-dependencies:${BIN_VERSION:-0.1.0} + target: compiler + + # Reuses the dependencies image as cache so we can parallelize binary builds + runtime: image: marcellodesales/cloner:${BIN_VERSION:-0.1.0} build: context: . args: - BIN_NAME=cloner - BIN_VERSION=${BIN_VERSION:-0.1.0} + - PLATFORMS=linux + cache_from: + - marcellodesales/cloner-binaries-linux:${BIN_VERSION:-0.1.0} + target: runtime From 7ebf42b8c6a5e9b61b877dcca85620ab2de08b23 Mon Sep 17 00:00:00 2001 From: Marcello de Sales Date: Tue, 22 Sep 2020 19:25:47 -0300 Subject: [PATCH 2/4] :hammer: Makefile: add constructs for parallel: docker BUILDKIT * build-dependencies: build the docker image with dependencies as cache to be reused by others. * compile: for specific language * runtime: for the linux runtime --- Makefile | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/Makefile b/Makefile index 6d664e6..d848055 100644 --- a/Makefile +++ b/Makefile @@ -40,6 +40,26 @@ build: clean ## Builds the docker image with binaries @echo "Building next version $(BIN_VERSION)" BIN_VERSION=$(BIN_VERSION) docker-compose build --build-arg BIN_VERSION=$(BIN_VERSION) cli +build-dependencies: clean ## Builds the docker image only with dependencies using buildkit + @echo "Building dependencies for version $(BIN_VERSION) - Dependencies ONLY" + DOCKER_BUILDKIT=1 BIN_VERSION=$(BIN_VERSION) docker-compose build dependencies + +compile-linux: build-dependencies ## Compiles for Linux + @echo "Compiling version $(BIN_VERSION) for linux" + DOCKER_BUILDKIT=1 BIN_VERSION=$(BIN_VERSION) PLATFORMS=linux docker-compose build binaries + +compile-darwin: build-dependencies ## Compiles for MacOS + @echo "Compiling version $(BIN_VERSION) for darwin" + DOCKER_BUILDKIT=1 BIN_VERSION=$(BIN_VERSION) PLATFORMS=darwin docker-compose build binaries + +compile-windows: build-dependencies ## Compiles for Windows + @echo "Compiling version $(BIN_VERSION) for windows" + DOCKER_BUILDKIT=1 BIN_VERSION=$(BIN_VERSION) PLATFORMS=windows docker-compose build binaries + +build-docker-runtime: + @echo "Building linux runtime for version $(BIN_VERSION)" + DOCKER_BUILDKIT=1 BIN_VERSION=$(BIN_VERSION) PLATFORMS=linux docker-compose build runtime + dist: build ## Makes the dir ./dist with binaries from docker image @echo "Distribution libraries for version $(BIN_VERSION)" docker run --rm --entrypoint sh -v $(PWD)/$(DIST_DIR):/bins $(ORG)/$(APP_NAME):$(BIN_VERSION) -c "cp /usr/local/bin/$(APP_NAME)-darwin-amd64 /bins" From adf787bd7576b275f42d60b3839b8df316e5a649 Mon Sep 17 00:00:00 2001 From: Marcello de Sales Date: Tue, 22 Sep 2020 21:17:10 -0300 Subject: [PATCH 3/4] :zap: develop Workflow: save dependencies docker image This is to speed up the compile task so that it makes images faster --- .github/workflows/develop.yaml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/.github/workflows/develop.yaml b/.github/workflows/develop.yaml index 8fa01e3..bf82db4 100644 --- a/.github/workflows/develop.yaml +++ b/.github/workflows/develop.yaml @@ -23,6 +23,33 @@ jobs: - name: Golang test run: make test + dependencies: + name: Build Dependencies + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + # https://github.com/actions/checkout/pull/258 needs to fetch all tags so that Makefile can make the correct version + fetch-depth: 0 + + - name: Build OS and Golang Dependencies + run: make build-dependencies + # outputs: the docker image with dependencies that can be used as cache for the binaries + # marcellodesales/cloner/dependencies:20.09.12 + + # https://github.community/t/cache-a-docker-image-built-in-workflow/16260/9 + # Produces the docker image at the directory ./dist/docker-image.raw + - name: Save Raw Docker Image for Dependencies + run: make save-dependencies-docker-image + # outputs: cloner-dependencies.dockerimage + + # Local cache of docker images + - name: Upload Docker Image for Dependencies + uses: actions/upload-artifact@v2 + with: + name: cloner.dockerimage + path: dist/cloner-dependencies.dockerimage + build: name: Build CLI Binaries runs-on: ubuntu-latest From 6c481b1c4c6c7796dc498cff707c29495937d2ff Mon Sep 17 00:00:00 2001 From: Marcello de Sales Date: Tue, 22 Sep 2020 23:23:44 -0300 Subject: [PATCH 4/4] :hammer: Workflows: build paralle docker images BUILDKIT --- .github/workflows/develop.yaml | 83 ++++++++++++++++++++-------------- Makefile | 29 ++++++++---- docker-compose.yaml | 10 ++-- 3 files changed, 74 insertions(+), 48 deletions(-) diff --git a/.github/workflows/develop.yaml b/.github/workflows/develop.yaml index bf82db4..7e77ad1 100644 --- a/.github/workflows/develop.yaml +++ b/.github/workflows/develop.yaml @@ -50,58 +50,75 @@ jobs: name: cloner.dockerimage path: dist/cloner-dependencies.dockerimage - build: - name: Build CLI Binaries - runs-on: ubuntu-latest + # https://github.com/nightlark/ninja/blob/f1a33131154ae7d9648aa82afac462859535fb62/.github/workflows/release-ninja-binaries.yml#L8-L34 + compile: + name: Compile CLI as Binaries + runs-on: ${{ matrix.os }} + needs: dependencies + strategy: + matrix: + os: [darwin, linux, windows] + include: + - os: darwin + extension: + - os: linux + extension: + - os: windows + extension: .exe steps: - uses: actions/checkout@v2 with: # https://github.com/actions/checkout/pull/258 needs to fetch all tags so that Makefile can make the correct version fetch-depth: 0 - - name: Dockerized Cross-compile Build - run: make build + - name: Download Docker Image for Dependencies as cache + uses: actions/download-artifact@v2 + with: + name: cloner-dependencies.dockerimage - # Produces the binaries at the directory ./dist - - name: Dockerized Binary Distribution - run: make dist + - name: Load Docker Image Dependencies for cache to compile + run: | + ls -la ./cloner.dockerimage + docker load -i ./cloner-dependencies.dockerimage - # https://github.community/t/cache-a-docker-image-built-in-workflow/16260/9 - # Produces the docker image at the directory ./dist/docker-image.raw - - name: Save Raw Docker Image for Reuse - run: make save-docker-image + # Compiles for OS specific dependencies through Dockerkit + - name: Compile for ${{matrix.os}} + env: + OS_NAME: ${{ matrix.os }} + run: make compile-${OS_NAME} - # https://docs.github.com/en/actions/configuring-and-managing-workflows/persisting-workflow-data-using-artifacts#passing-data-between-jobs-in-a-workflow - - name: Upload MacOS Binary - uses: actions/upload-artifact@v2 - with: - name: cloner-darwin-amd64 - path: dist/cloner-darwin-amd64 + # Compiles for OS specific dependencies through Dockerkit + - name: Get distribution binaries for ${{matrix.os}} + env: + OS_NAME: ${{ matrix.os }} + run: make dist-${OS_NAME} - - name: Upload Linux Binary + - name: Upload ${{ matrix.os }} Binary uses: actions/upload-artifact@v2 with: - name: cloner-linux-amd64 - path: dist/cloner-linux-amd64 + name: cloner-${{ matrix.os }}-amd64.${{ matrix.extension }} + path: dist/cloner-${{ matrix.os }}-amd64.${{ matrix.extension }} - - name: Upload Windows Binary - uses: actions/upload-artifact@v2 - with: - name: cloner-windows-amd64.exe - path: dist/cloner-windows-amd64.exe + # Compiles for OS specific dependencies through Dockerkit + - name: Save docker runtime image for ${{matrix.os}} + if: matrix.os == 'linux' + env: + OS_NAME: ${{ matrix.os }} + run: make save-docker-image + # output: docker image saved at ./dist/cloner.dockerimage - # Local cache of docker images - - name: Upload Docker Image + - name: Upload runtime docker image uses: actions/upload-artifact@v2 + if: matrix.os == 'linux' with: name: cloner.dockerimage - path: dist/cloner.dockerimage + path: ./dist/cloner.dockerimage # https://github.com/nightlark/ninja/blob/f1a33131154ae7d9648aa82afac462859535fb62/.github/workflows/release-ninja-binaries.yml#L8-L34 verify: name: Verify CLI Binaries runs-on: ${{ matrix.os }} - needs: build + needs: compile strategy: matrix: os: [ubuntu-latest, macOS-latest, windows-latest] @@ -133,7 +150,7 @@ jobs: if: matrix.os == 'windows-latest' env: BIN_NAME: ${{ matrix.bin_name }} - # https://stackoverflow.com/questions/53961802/how-to-use-an-environment-variable-in-powershell-command/53963070#53963070 + # start-process -nonewwindow https://stackoverflow.com/questions/53961802/how-to-use-an-environment-variable-in-powershell-command/53963070#53963070 run: | dir echo $pwd\$env:BIN_NAME @@ -142,7 +159,7 @@ jobs: e2e: name: Verify Dockerized E2E Test runs-on: ubuntu-latest - needs: build + needs: compile steps: - uses: actions/checkout@v2 with: @@ -173,7 +190,7 @@ jobs: push: name: Push CLI Docker Images runs-on: ubuntu-latest - needs: build + needs: compile steps: - uses: actions/checkout@v2 with: diff --git a/Makefile b/Makefile index d848055..46c450b 100644 --- a/Makefile +++ b/Makefile @@ -44,22 +44,31 @@ build-dependencies: clean ## Builds the docker image only with dependencies usin @echo "Building dependencies for version $(BIN_VERSION) - Dependencies ONLY" DOCKER_BUILDKIT=1 BIN_VERSION=$(BIN_VERSION) docker-compose build dependencies -compile-linux: build-dependencies ## Compiles for Linux - @echo "Compiling version $(BIN_VERSION) for linux" - DOCKER_BUILDKIT=1 BIN_VERSION=$(BIN_VERSION) PLATFORMS=linux docker-compose build binaries - -compile-darwin: build-dependencies ## Compiles for MacOS - @echo "Compiling version $(BIN_VERSION) for darwin" - DOCKER_BUILDKIT=1 BIN_VERSION=$(BIN_VERSION) PLATFORMS=darwin docker-compose build binaries +save-dependencies-docker-image: ## Saves the raw docker image of dependencies for cache +ifndef GITHUB_ACTION + $(error GITHUB_ACTION is undefined. This must run only by Github Actions) +endif + $(eval BUILD_IMAGE_TAG=$(shell BIN_VERSION=$(BIN_VERSION) docker-compose config | grep image: | grep dependencies | awk '{print $$2}')) + docker save -o ./dist/$(APP_NAME)-dependencies.dockerimage $(BUILD_IMAGE_TAG) + ls -la ./dist/$(APP_NAME)-dependencies.dockerimage -compile-windows: build-dependencies ## Compiles for Windows - @echo "Compiling version $(BIN_VERSION) for windows" - DOCKER_BUILDKIT=1 BIN_VERSION=$(BIN_VERSION) PLATFORMS=windows docker-compose build binaries +compile-%: build-dependencies ## Compiles for (darwin, linux, windows) + $(eval PLATFORM=$(shell echo $@ | awk -F"-" '{print $$2}')) + @echo "Compiling version $(BIN_VERSION) for $(PLATFORM)" + DOCKER_BUILDKIT=1 BIN_VERSION=$(BIN_VERSION) PLATFORMS=$(PLATFORM) docker-compose build binaries build-docker-runtime: @echo "Building linux runtime for version $(BIN_VERSION)" DOCKER_BUILDKIT=1 BIN_VERSION=$(BIN_VERSION) PLATFORMS=linux docker-compose build runtime +dist-%: ## Makes the dir ./dist with binaries from docker image + $(eval PLATFORM=$(shell echo $@ | awk -F"-" '{print $$2}')) + @echo "$(PLATFORM) Distribution binary for version $(BIN_VERSION)" + $(eval DIST_IMAGE=$(shell DOCKER_BUILDKIT=1 BIN_VERSION=$(BIN_VERSION) PLATFORMS=$(PLATFORM) docker-compose config | grep image: | grep binaries | awk '{print $$2}')) + docker run --rm --entrypoint sh -v $(PWD)/$(DIST_DIR):/bins $(DIST_IMAGE) -c "cp /build/$(APP_NAME)-$(PLATFORM)-amd64s /bins" || true + docker run --rm --entrypoint sh -v $(PWD)/$(DIST_DIR):/bins $(DIST_IMAGE) -c "cp /build/$(APP_NAME)-$(PLATFORM)-amd64.exe /bins" || true + ls -la $(PWD)/$(DIST_DIR) + dist: build ## Makes the dir ./dist with binaries from docker image @echo "Distribution libraries for version $(BIN_VERSION)" docker run --rm --entrypoint sh -v $(PWD)/$(DIST_DIR):/bins $(ORG)/$(APP_NAME):$(BIN_VERSION) -c "cp /usr/local/bin/$(APP_NAME)-darwin-amd64 /bins" diff --git a/docker-compose.yaml b/docker-compose.yaml index 56101a8..032db30 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -4,22 +4,22 @@ services: # Usually takes time to download, and can be reused for all platforms dependencies: - image: marcellodesales/cloner-dependencies:${BIN_VERSION:-0.1.0} + image: marcellodesales/cloner/dependencies:${BIN_VERSION:-0.1.0} build: context: . target: dependencies # Reuses the dependencies image as cache so we can parallelize binary builds binaries: - image: marcellodesales/cloner-binaries-${PLATFORMS}:${BIN_VERSION:-0.1.0} + image: marcellodesales/cloner/binaries-${PLATFORMS:-all}:${BIN_VERSION:-0.1.0} build: context: . args: - BIN_NAME=cloner - BIN_VERSION=${BIN_VERSION:-0.1.0} - - PLATFORMS=${PLATFORMS} + - PLATFORMS=${PLATFORMS:-darwin linux windows} cache_from: - - marcellodesales/cloner-dependencies:${BIN_VERSION:-0.1.0} + - marcellodesales/cloner/dependencies:${BIN_VERSION:-0.1.0} target: compiler # Reuses the dependencies image as cache so we can parallelize binary builds @@ -32,5 +32,5 @@ services: - BIN_VERSION=${BIN_VERSION:-0.1.0} - PLATFORMS=linux cache_from: - - marcellodesales/cloner-binaries-linux:${BIN_VERSION:-0.1.0} + - marcellodesales/cloner/binaries-linux:${BIN_VERSION:-0.1.0} target: runtime