diff --git a/.github/workflows/develop.yaml b/.github/workflows/develop.yaml index 8fa01e3..7e77ad1 100644 --- a/.github/workflows/develop.yaml +++ b/.github/workflows/develop.yaml @@ -23,8 +23,8 @@ jobs: - name: Golang test run: make test - build: - name: Build CLI Binaries + dependencies: + name: Build Dependencies runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 @@ -32,49 +32,93 @@ jobs: # https://github.com/actions/checkout/pull/258 needs to fetch all tags so that Makefile can make the correct version fetch-depth: 0 - - name: Dockerized Cross-compile Build - run: make build - - # Produces the binaries at the directory ./dist - - name: Dockerized Binary Distribution - run: make dist + - name: Build OS and Golang Dependencies + run: make build-dependencies + # outputs: the docker image with dependencies that can be used as cache for the binaries + # marcellodesales/cloner/dependencies:20.09.12 # https://github.community/t/cache-a-docker-image-built-in-workflow/16260/9 # Produces the docker image at the directory ./dist/docker-image.raw - - name: Save Raw Docker Image for Reuse - run: make save-docker-image + - name: Save Raw Docker Image for Dependencies + run: make save-dependencies-docker-image + # outputs: cloner-dependencies.dockerimage - # https://docs.github.com/en/actions/configuring-and-managing-workflows/persisting-workflow-data-using-artifacts#passing-data-between-jobs-in-a-workflow - - name: Upload MacOS Binary + # Local cache of docker images + - name: Upload Docker Image for Dependencies uses: actions/upload-artifact@v2 with: - name: cloner-darwin-amd64 - path: dist/cloner-darwin-amd64 + name: cloner.dockerimage + path: dist/cloner-dependencies.dockerimage - - name: Upload Linux Binary - uses: actions/upload-artifact@v2 + # https://github.com/nightlark/ninja/blob/f1a33131154ae7d9648aa82afac462859535fb62/.github/workflows/release-ninja-binaries.yml#L8-L34 + compile: + name: Compile CLI as Binaries + runs-on: ${{ matrix.os }} + needs: dependencies + strategy: + matrix: + os: [darwin, linux, windows] + include: + - os: darwin + extension: + - os: linux + extension: + - os: windows + extension: .exe + steps: + - uses: actions/checkout@v2 + with: + # https://github.com/actions/checkout/pull/258 needs to fetch all tags so that Makefile can make the correct version + fetch-depth: 0 + + - name: Download Docker Image for Dependencies as cache + uses: actions/download-artifact@v2 with: - name: cloner-linux-amd64 - path: dist/cloner-linux-amd64 + name: cloner-dependencies.dockerimage + + - name: Load Docker Image Dependencies for cache to compile + run: | + ls -la ./cloner.dockerimage + docker load -i ./cloner-dependencies.dockerimage + + # Compiles for OS specific dependencies through Dockerkit + - name: Compile for ${{matrix.os}} + env: + OS_NAME: ${{ matrix.os }} + run: make compile-${OS_NAME} + + # Compiles for OS specific dependencies through Dockerkit + - name: Get distribution binaries for ${{matrix.os}} + env: + OS_NAME: ${{ matrix.os }} + run: make dist-${OS_NAME} - - name: Upload Windows Binary + - name: Upload ${{ matrix.os }} Binary uses: actions/upload-artifact@v2 with: - name: cloner-windows-amd64.exe - path: dist/cloner-windows-amd64.exe + name: cloner-${{ matrix.os }}-amd64.${{ matrix.extension }} + path: dist/cloner-${{ matrix.os }}-amd64.${{ matrix.extension }} - # Local cache of docker images - - name: Upload Docker Image + # Compiles for OS specific dependencies through Dockerkit + - name: Save docker runtime image for ${{matrix.os}} + if: matrix.os == 'linux' + env: + OS_NAME: ${{ matrix.os }} + run: make save-docker-image + # output: docker image saved at ./dist/cloner.dockerimage + + - name: Upload runtime docker image uses: actions/upload-artifact@v2 + if: matrix.os == 'linux' with: name: cloner.dockerimage - path: dist/cloner.dockerimage + path: ./dist/cloner.dockerimage # https://github.com/nightlark/ninja/blob/f1a33131154ae7d9648aa82afac462859535fb62/.github/workflows/release-ninja-binaries.yml#L8-L34 verify: name: Verify CLI Binaries runs-on: ${{ matrix.os }} - needs: build + needs: compile strategy: matrix: os: [ubuntu-latest, macOS-latest, windows-latest] @@ -106,7 +150,7 @@ jobs: if: matrix.os == 'windows-latest' env: BIN_NAME: ${{ matrix.bin_name }} - # https://stackoverflow.com/questions/53961802/how-to-use-an-environment-variable-in-powershell-command/53963070#53963070 + # start-process -nonewwindow https://stackoverflow.com/questions/53961802/how-to-use-an-environment-variable-in-powershell-command/53963070#53963070 run: | dir echo $pwd\$env:BIN_NAME @@ -115,7 +159,7 @@ jobs: e2e: name: Verify Dockerized E2E Test runs-on: ubuntu-latest - needs: build + needs: compile steps: - uses: actions/checkout@v2 with: @@ -146,7 +190,7 @@ jobs: push: name: Push CLI Docker Images runs-on: ubuntu-latest - needs: build + needs: compile steps: - uses: actions/checkout@v2 with: diff --git a/Dockerfile b/Dockerfile index 75bc7cd..dff3d8c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ #first stage - builder -FROM golang:1.13.0-stretch as builder +FROM golang:1.13.0-stretch AS dependencies WORKDIR /build @@ -18,6 +18,8 @@ COPY go.sum /build/go.sum ENV GO111MODULE=on RUN go mod download +FROM dependencies AS compiler + # Add the main COPY main.go /build/main.go @@ -33,11 +35,13 @@ COPY util /build/util ARG BIN_NAME ARG BIN_VERSION +ARG PLATFORMS # Cross-compile all versions ENV BIN_NAME ${BIN_NAME:-unknown} ENV BUILD_VERSION ${BIN_VERSION:-0.1.0} -ENV PLATFORMS "darwin linux windows" +ENV PLATFORMS ${PLATFORMS:-darwin linux windows} +RUN echo "Building for ${PLATFORMS}" ENV ARCHS "amd64" #ENV ARCHS "386 amd64" @@ -58,14 +62,14 @@ RUN export export FULL_NAME_GIT=$(git -C /build/.git remote -v | grep fetch | aw #RUN upx --lzma /build/${BIN_NAME}* # Build the main container (Linux Runtime) -FROM alpine:latest +FROM alpine:latest AS runtime WORKDIR /root/ ARG BIN_NAME ENV BIN_NAME ${BIN_NAME:-unknown} # Copy the linux amd64 binary, based on the arg (or else all files are copied) inspect with https://github.com/wagoodman/dive -COPY --from=builder /build/${BIN_NAME}* /usr/local/bin/ +COPY --from=compiler /build/${BIN_NAME}* /usr/local/bin/ # Move the bin to /usr/local/bin and make the entrypoint to point to it passing the params # https://stackoverflow.com/questions/33439230/how-to-write-commands-with-multiple-lines-in-dockerfile-while-preserving-the-new/33439625#33439625 diff --git a/Makefile b/Makefile index 6d664e6..46c450b 100644 --- a/Makefile +++ b/Makefile @@ -40,6 +40,35 @@ build: clean ## Builds the docker image with binaries @echo "Building next version $(BIN_VERSION)" BIN_VERSION=$(BIN_VERSION) docker-compose build --build-arg BIN_VERSION=$(BIN_VERSION) cli +build-dependencies: clean ## Builds the docker image only with dependencies using buildkit + @echo "Building dependencies for version $(BIN_VERSION) - Dependencies ONLY" + DOCKER_BUILDKIT=1 BIN_VERSION=$(BIN_VERSION) docker-compose build dependencies + +save-dependencies-docker-image: ## Saves the raw docker image of dependencies for cache +ifndef GITHUB_ACTION + $(error GITHUB_ACTION is undefined. This must run only by Github Actions) +endif + $(eval BUILD_IMAGE_TAG=$(shell BIN_VERSION=$(BIN_VERSION) docker-compose config | grep image: | grep dependencies | awk '{print $$2}')) + docker save -o ./dist/$(APP_NAME)-dependencies.dockerimage $(BUILD_IMAGE_TAG) + ls -la ./dist/$(APP_NAME)-dependencies.dockerimage + +compile-%: build-dependencies ## Compiles for (darwin, linux, windows) + $(eval PLATFORM=$(shell echo $@ | awk -F"-" '{print $$2}')) + @echo "Compiling version $(BIN_VERSION) for $(PLATFORM)" + DOCKER_BUILDKIT=1 BIN_VERSION=$(BIN_VERSION) PLATFORMS=$(PLATFORM) docker-compose build binaries + +build-docker-runtime: + @echo "Building linux runtime for version $(BIN_VERSION)" + DOCKER_BUILDKIT=1 BIN_VERSION=$(BIN_VERSION) PLATFORMS=linux docker-compose build runtime + +dist-%: ## Makes the dir ./dist with binaries from docker image + $(eval PLATFORM=$(shell echo $@ | awk -F"-" '{print $$2}')) + @echo "$(PLATFORM) Distribution binary for version $(BIN_VERSION)" + $(eval DIST_IMAGE=$(shell DOCKER_BUILDKIT=1 BIN_VERSION=$(BIN_VERSION) PLATFORMS=$(PLATFORM) docker-compose config | grep image: | grep binaries | awk '{print $$2}')) + docker run --rm --entrypoint sh -v $(PWD)/$(DIST_DIR):/bins $(DIST_IMAGE) -c "cp /build/$(APP_NAME)-$(PLATFORM)-amd64s /bins" || true + docker run --rm --entrypoint sh -v $(PWD)/$(DIST_DIR):/bins $(DIST_IMAGE) -c "cp /build/$(APP_NAME)-$(PLATFORM)-amd64.exe /bins" || true + ls -la $(PWD)/$(DIST_DIR) + dist: build ## Makes the dir ./dist with binaries from docker image @echo "Distribution libraries for version $(BIN_VERSION)" docker run --rm --entrypoint sh -v $(PWD)/$(DIST_DIR):/bins $(ORG)/$(APP_NAME):$(BIN_VERSION) -c "cp /usr/local/bin/$(APP_NAME)-darwin-amd64 /bins" diff --git a/docker-compose.yaml b/docker-compose.yaml index 42357d2..032db30 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,11 +1,36 @@ -version: "3" +version: "3.8" services: - cli: + # Usually takes time to download, and can be reused for all platforms + dependencies: + image: marcellodesales/cloner/dependencies:${BIN_VERSION:-0.1.0} + build: + context: . + target: dependencies + + # Reuses the dependencies image as cache so we can parallelize binary builds + binaries: + image: marcellodesales/cloner/binaries-${PLATFORMS:-all}:${BIN_VERSION:-0.1.0} + build: + context: . + args: + - BIN_NAME=cloner + - BIN_VERSION=${BIN_VERSION:-0.1.0} + - PLATFORMS=${PLATFORMS:-darwin linux windows} + cache_from: + - marcellodesales/cloner/dependencies:${BIN_VERSION:-0.1.0} + target: compiler + + # Reuses the dependencies image as cache so we can parallelize binary builds + runtime: image: marcellodesales/cloner:${BIN_VERSION:-0.1.0} build: context: . args: - BIN_NAME=cloner - BIN_VERSION=${BIN_VERSION:-0.1.0} + - PLATFORMS=linux + cache_from: + - marcellodesales/cloner/binaries-linux:${BIN_VERSION:-0.1.0} + target: runtime