Merge pull request '[CI] Forgejo Actions based release process' (#861) from wip-release into forgejo-ci

Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/861

.forgejo/actions/build-release/action.yml

@@ -0,0 +1,144 @@
+name: 'Build release'
+author: 'Forgejo authors'
+description: |
+  Build release
+
+inputs:
+  forgejo:
+    description: 'URL of the Forgejo instance where the release is uploaded'
+    required: true
+  owner:
+    description: 'User or organization where the release is uploaded, relative to the Forgejo instance'
+    required: true
+  repository:
+    description: 'Repository where the release is uploaded, relative to the owner'
+    required: true
+  doer:
+    description: 'Name of the user authoring the release'
+    required: true
+  ref_name:
+    description: 'Git reference of the tag for which the release must be built'
+    required: true
+  suffix:
+    description: 'Suffix to add to the image tag'
+  token:
+    description: 'token'
+    required: true
+  dockerfile:
+    description: 'path to the dockerfile'
+    default: 'Dockerfile'
+  platforms:
+    description: 'Coma separated list of platforms'
+    default: 'linux/amd64,linux/arm64'
+  binary-name:
+    description: 'Name of the binary'
+  binary-path:
+    description: 'Path of the binary within the container to extract into binary-name'
+  verbose:
+    description: 'Increase the verbosity level'
+    default: 'false'
+
+runs:
+  using: "composite"
+  steps:
+    - run: echo "${{ github.action_path }}" >> $GITHUB_PATH
+      shell: bash
+
+    - name: set -x if verbose is required
+      id: verbose
+      run: |
+        if ${{ inputs.verbose }} ; then
+          echo "shell=set -x" >> "$GITHUB_OUTPUT"
+        fi
+
+    - name: Create the insecure and buildx-config variables for the container registry
+      id: registry
+      run: |
+        ${{ steps.verbose.outputs.shell }}
+        url="${{ inputs.forgejo }}"
+        hostport=${url##http*://}
+        hostport=${hostport%%/}
+        echo "host-port=${hostport}" >> "$GITHUB_OUTPUT"
+        if ! [[ $url =~ ^http:// ]] ; then
+           exit 0
+        fi
+        cat >> "$GITHUB_OUTPUT" <<EOF
+        insecure=true
+        buildx-config<<ENDVAR
+        [registry."${hostport}"]
+          http = true
+        ENDVAR
+        EOF
+
+    - name: Allow docker pull/push to forgejo
+      if: ${{ steps.registry.outputs.insecure }}
+      run: |-
+        mkdir -p /etc/docker
+        cat > /etc/docker/daemon.json <<EOF
+          {
+            "insecure-registries" : ["${{ steps.registry.outputs.host-port }}"],
+            "bip": "172.26.0.1/16"
+          }
+        EOF
+
+    - name: Install docker
+      run: |
+        echo deb http://deb.debian.org/debian bullseye-backports main | tee /etc/apt/sources.list.d/backports.list && apt-get -qq update
+        DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -qq -y -t bullseye-backports docker.io
+
+    - uses: https://github.com/docker/setup-buildx-action@v2
+      with:
+        config-inline: |
+         ${{ steps.registry.outputs.buildx-config }}
+
+    - name: Login to the container registry
+      run: |
+        BASE64_AUTH=`echo -n "${{ inputs.doer }}:${{ inputs.token }}" | base64`
+        mkdir -p ~/.docker
+        echo "{\"auths\": {\"$CI_REGISTRY\": {\"auth\": \"$BASE64_AUTH\"}}}" > ~/.docker/config.json
+      env:
+        CI_REGISTRY: "${{ steps.registry.outputs.host-port }}"
+
+    - name: Create the tag and image variable
+      id: build
+      run: |
+        ${{ steps.verbose.outputs.shell }}
+        tag="${{ inputs.ref_name }}"
+        tag=${tag##*v}
+        echo "tag=$tag" >> "$GITHUB_OUTPUT"
+
+    - name: Build the container image for each architecture
+      uses: https://github.com/docker/build-push-action@v4
+      # workaround until https://github.com/docker/build-push-action/commit/d8823bfaed2a82c6f5d4799a2f8e86173c461aba is in @v4 or @v5 is released
+      env:
+        ACTIONS_RUNTIME_TOKEN: ''
+      with:
+        context: .
+        push: true
+        file: ${{ inputs.dockerfile }}
+        platforms: ${{ inputs.platforms }}
+        tags: ${{ steps.registry.outputs.host-port }}/${{ inputs.owner }}/${{ inputs.repository }}:${{ steps.build.outputs.tag }}${{ inputs.suffix }}
+
+    - name: Extract the binary from the container images into the release directory
+      if: inputs.binary-name != ''
+      run: |
+        ${{ steps.verbose.outputs.shell }}
+        mkdir -p release
+        for platform in $(echo ${{ inputs.platforms }} | tr ',' ' '); do
+          arch=$(echo $platform | sed -e 's|linux/||g' -e 's|arm/v6|arm-6|g')
+          docker create --platform $platform --name forgejo-$arch ${{ steps.registry.outputs.host-port }}/${{ inputs.owner }}/${{ inputs.repository }}:${{ steps.build.outputs.tag }}${{ inputs.suffix }}
+          binary="${{ inputs.binary-name }}-${{ steps.build.outputs.tag }}"
+          docker cp forgejo-$arch:${{ inputs.binary-path }} release/$binary-$arch
+          shasum -a 256 < release/$binary-$arch | cut -f1 -d ' ' > release/$binary-$arch.sha256
+          docker rm forgejo-$arch
+        done
+
+    - name: publish release
+      if: inputs.binary-name != ''
+      uses: https://code.forgejo.org/actions/forgejo-release@v1
+      with:
+        direction: upload
+        release-dir: release
+        release-notes: "RELEASE-NOTES#${{ steps.build.outputs.tag }}"
+        token: ${{ inputs.token }}
+        verbose: ${{ steps.verbose.outputs.value }}

.forgejo/workflows/build-release-integration.yml

@@ -0,0 +1,86 @@
+name: Integration tests for the release process
+
+on:
+  push:
+    paths:
+      - Makefile
+      - docker/**
+      - .forgejo/actions/build-release/action.yml
+      - .forgejo/workflows/build-release.yml
+      - .forgejo/workflows/build-release-integration.yml
+
+jobs:
+  release-simulation:
+    runs-on: self-hosted
+    if: secrets.ROLE != 'forgejo-integration' && secrets.ROLE != 'forgejo-experimental' && secrets.ROLE != 'forgejo-release'
+    steps:
+      - uses: actions/checkout@v3
+
+      - id: forgejo
+        uses: https://code.forgejo.org/actions/setup-forgejo@v1
+        with:
+          user: root
+          password: admin1234
+          image-version: 1.19
+          lxc-ip-prefix: 10.0.9
+
+      - name: publish the forgejo release
+        run: |
+          set -x
+
+          cat > /etc/docker/daemon.json <<EOF
+            {
+              "insecure-registries" : ["${{ steps.forgejo.outputs.host-port }}"]
+            }
+          EOF
+          systemctl restart docker
+
+          dir=$(mktemp -d)
+          trap "rm -fr $dir" EXIT
+
+          url=http://root:admin1234@${{ steps.forgejo.outputs.host-port }}
+          export FORGEJO_RUNNER_LOGS="${{ steps.forgejo.outputs.runner-logs }}"
+
+          #
+          # Create a new project with a fake forgejo and the release workflow only
+          #
+          mkdir -p $dir/.forgejo/workflows
+          cp .forgejo/workflows/build-release.yml $dir/.forgejo/workflows
+          cp -a .forgejo/actions $dir/.forgejo/actions
+          cat > $dir/Dockerfile <<EOF
+          FROM docker.io/library/alpine:3.18
+          RUN mkdir -p /app/gitea
+          RUN ( echo '#!/bin/sh' ; echo 'echo forgejo v1.2.3' ) > /app/gitea/gitea ; chmod +x /app/gitea/gitea
+          EOF
+          cp $dir/Dockerfile $dir/Dockerfile.rootless
+
+          forgejo-test-helper.sh push $dir $url root forgejo |& tee $dir/pushed
+          eval $(grep '^sha=' < $dir/pushed)
+
+          #
+          # Push a tag to trigger the release workflow and wait for it to complete
+          #
+          forgejo-test-helper.sh api POST $url repos/root/forgejo/tags ${{ steps.forgejo.outputs.token }} --data-raw '{"tag_name": "v1.2.3", "target": "'$sha'"}'
+          LOOPS=180 forgejo-test-helper.sh wait_success "$url" root/forgejo $sha
+
+          #
+          # uncomment to see the logs even when everything is reported to be working ok
+          #
+          cat $FORGEJO_RUNNER_LOGS
+
+          #
+          # Minimal sanity checks. e2e test is for the setup-forgejo
+          # action and the infrastructure playbook. Since the binary
+          # is a script shell it does not test the sanity of the cross
+          # build, only the sanity of the naming of the binaries.
+          #
+          for arch in amd64 arm64 arm-6 ; do
+            curl -L -sS $url/root/forgejo/releases/download/v1.2.3/forgejo-1.2.3-$arch > forgejo
+            chmod +x forgejo
+            ./forgejo --version | grep 1.2.3
+            curl -L -sS $url/root/forgejo/releases/download/v1.2.3/forgejo-1.2.3-$arch.sha256 > forgejo.one
+            shasum -a 256 < forgejo | cut -f1 -d ' ' > forgejo.two
+            diff forgejo.one forgejo.two
+          done
+          docker pull ${{ steps.forgejo.outputs.host-port }}/root/forgejo:1.2.3
+          docker pull ${{ steps.forgejo.outputs.host-port }}/root/forgejo:1.2.3-rootless

.forgejo/workflows/build-release.yml

@@ -0,0 +1,104 @@
+name: Build release
+
+on:
+  push:
+    tags: 'v*'
+
+jobs:
+  release:
+    runs-on: self-hosted
+    # root is used for testing, allow it
+    if: secrets.ROLE == 'forgejo-integration' || github.repository_owner == 'root'
+    steps:
+      - uses: actions/checkout@v3
+
+      - id: verbose
+        run: |
+          # if there are no secrets, be verbose
+          if test -z "${{ secrets.TOKEN }}"; then
+            value=true
+          else
+            value=false
+          fi
+          echo "value=$value" >> "$GITHUB_OUTPUT"
+
+      - id: repository
+        run: |
+          set -x # comment out
+          repository="${{ github.repository }}"
+          echo "value=${repository##*/}" >> "$GITHUB_OUTPUT"
+
+      - name: when in a test environment, create a token
+        id: token
+        if: ${{ secrets.TOKEN == '' }}
+        run: |
+          set -x # comment out
+          apt-get -qq install -y jq
+          url="${{ env.GITHUB_SERVER_URL }}"
+          hostport=${url##http*://}
+          hostport=${hostport%%/}
+          doer=root
+          api=http://$doer:admin1234@$hostport/api/v1/users/$doer/tokens
+          curl -sS -X DELETE $api/release
+          token=$(curl -sS -X POST -H 'Content-Type: application/json' --data-raw '{"name": "release", "scopes": ["all"]}' $api | jq --raw-output .sha1)
+          echo "value=${token}" >> "$GITHUB_OUTPUT"
+
+      - name: build container & release (when TOKEN secret is not set)
+        if: ${{ secrets.TOKEN == '' }}
+        uses: ./.forgejo/actions/build-release
+        with:
+          forgejo: "${{ env.GITHUB_SERVER_URL }}"
+          owner: "${{ env.GITHUB_REPOSITORY_OWNER }}"
+          repository: "${{ steps.repository.outputs.value }}"
+          doer: root
+          ref_name: "${{ github.ref_name }}"
+          token: ${{ steps.token.outputs.value }}
+          platforms: linux/amd64,linux/arm64,linux/arm/v6
+          binary-name: forgejo
+          binary-path: /app/gitea/gitea
+          verbose: ${{ steps.verbose.outputs.value }}
+
+      - name: build rootless container (when TOKEN secret is not set)
+        if: ${{ secrets.TOKEN == '' }}
+        uses: ./.forgejo/actions/build-release
+        with:
+          forgejo: "${{ env.GITHUB_SERVER_URL }}"
+          owner: "${{ env.GITHUB_REPOSITORY_OWNER }}"
+          repository: "${{ steps.repository.outputs.value }}"
+          doer: root
+          ref_name: "${{ github.ref_name }}"
+          token: ${{ steps.token.outputs.value }}
+          platforms: linux/amd64,linux/arm64,linux/arm/v6
+          suffix: -rootless
+          dockerfile: Dockerfile.rootless
+          verbose: ${{ steps.verbose.outputs.value }}
+
+      - name: build container & release (when TOKEN secret is set)
+        if: ${{ secrets.TOKEN != '' }}
+        uses: ./.forgejo/actions/build-release
+        with:
+          forgejo: "${{ env.GITHUB_SERVER_URL }}"
+          owner: "${{ env.GITHUB_REPOSITORY_OWNER }}"
+          repository: "${{ steps.repository.outputs.value }}"
+          doer: "${{ secrets.DOER }}"
+          ref_name: "${{ github.ref_name }}"
+          token: "${{ secrets.TOKEN }}"
+          platforms: linux/amd64,linux/arm64,linux/arm/v6
+          binary-name: forgejo
+          binary-path: /app/gitea/gitea
+          verbose: ${{ steps.verbose.outputs.value }}
+
+      - name: build rootless container (when TOKEN secret is set)
+        if: ${{ secrets.TOKEN != '' }}
+        uses: ./.forgejo/actions/build-release
+        with:
+          forgejo: "${{ env.GITHUB_SERVER_URL }}"
+          owner: "${{ env.GITHUB_REPOSITORY_OWNER }}"
+          repository: "${{ steps.repository.outputs.value }}"
+          doer: "${{ secrets.DOER }}"
+          ref_name: "${{ github.ref_name }}"
+          token: "${{ secrets.TOKEN }}"
+          platforms: linux/amd64,linux/arm64,linux/arm/v6
+          suffix: -rootless
+          dockerfile: Dockerfile.rootless
+          verbose: ${{ steps.verbose.outputs.value }}

Dockerfile

@@ -1,5 +1,6 @@
-#Build stage
+FROM --platform=$BUILDPLATFORM tonistiigi/xx AS xx
-FROM docker.io/library/golang:1.20-alpine3.18 AS build-env
+
+FROM --platform=$BUILDPLATFORM docker.io/library/golang:1.20-alpine3.18 as build-env
 
 ARG GOPROXY
 ENV GOPROXY ${GOPROXY:-direct}
@@ -9,19 +10,33 @@ ARG TAGS="sqlite sqlite_unlock_notify"
 ENV TAGS "bindata timetzdata $TAGS"
 ARG CGO_EXTRA_CFLAGS
 
-#Build deps
+#
+# Transparently cross compile for the target platform
+#
+COPY --from=xx / /
+ARG TARGETPLATFORM
+RUN apk --no-cache add clang lld
+RUN xx-apk --no-cache add gcc musl-dev
+ENV CGO_ENABLED=1
+RUN xx-go --wrap
+#
+# for go generate and binfmt to find
+# without it the generate phase will fail with
+# #19 25.04 modules/public/public_bindata.go:8: running "go": exit status 1
+# #19 25.39 aarch64-binfmt-P: Could not open '/lib/ld-musl-aarch64.so.1': No such file or directory
+# why exactly is it needed? where is binfmt involved?
+#
+RUN cp /*-alpine-linux-musl*/lib/ld-musl-*.so.1 /lib || true
+
 RUN apk --no-cache add build-base git nodejs npm
 
-#Setup repo
 COPY . ${GOPATH}/src/code.gitea.io/gitea
 WORKDIR ${GOPATH}/src/code.gitea.io/gitea
 
-#Checkout version if set
+RUN make clean-all
-RUN if [ -n "${GITEA_VERSION}" ]; then git checkout "${GITEA_VERSION}"; fi \
+RUN make frontend
- && make clean-all build
+RUN go build contrib/environment-to-ini/environment-to-ini.go && xx-verify environment-to-ini
-
+RUN make backend && xx-verify gitea
-# Begin env-to-ini build
-RUN go build contrib/environment-to-ini/environment-to-ini.go
 
 FROM docker.io/library/alpine:3.18
 LABEL maintainer="contact@forgejo.org"

Dockerfile.rootless

@@ -1,5 +1,6 @@
-#Build stage
+FROM --platform=$BUILDPLATFORM tonistiigi/xx AS xx
-FROM docker.io/library/golang:1.20-alpine3.18 AS build-env
+
+FROM --platform=$BUILDPLATFORM docker.io/library/golang:1.20-alpine3.18 as build-env
 
 ARG GOPROXY
 ENV GOPROXY ${GOPROXY:-direct}
@@ -9,19 +10,33 @@ ARG TAGS="sqlite sqlite_unlock_notify"
 ENV TAGS "bindata timetzdata $TAGS"
 ARG CGO_EXTRA_CFLAGS
 
-#Build deps
+#
+# Transparently cross compile for the target platform
+#
+COPY --from=xx / /
+ARG TARGETPLATFORM
+RUN apk --no-cache add clang lld
+RUN xx-apk --no-cache add gcc musl-dev
+ENV CGO_ENABLED=1
+RUN xx-go --wrap
+#
+# for go generate and binfmt to find
+# without it the generate phase will fail with
+# #19 25.04 modules/public/public_bindata.go:8: running "go": exit status 1
+# #19 25.39 aarch64-binfmt-P: Could not open '/lib/ld-musl-aarch64.so.1': No such file or directory
+# why exactly is it needed? where is binfmt involved?
+#
+RUN cp /*-alpine-linux-musl*/lib/ld-musl-*.so.1 /lib || true
+
 RUN apk --no-cache add build-base git nodejs npm
 
-#Setup repo
 COPY . ${GOPATH}/src/code.gitea.io/gitea
 WORKDIR ${GOPATH}/src/code.gitea.io/gitea
 
-#Checkout version if set
+RUN make clean-all
-RUN if [ -n "${GITEA_VERSION}" ]; then git checkout "${GITEA_VERSION}"; fi \
+RUN make frontend
- && make clean-all build
+RUN go build contrib/environment-to-ini/environment-to-ini.go && xx-verify environment-to-ini
-
+RUN make backend && xx-verify gitea
-# Begin env-to-ini build
-RUN go build contrib/environment-to-ini/environment-to-ini.go
 
 FROM docker.io/library/alpine:3.18
 LABEL maintainer="contact@forgejo.org"

Makefile

@@ -816,7 +816,7 @@ security-check:
 	go run $(GOVULNCHECK_PACKAGE) ./...
 
 $(EXECUTABLE): $(GO_SOURCES) $(TAGS_PREREQ)
-	CGO_CFLAGS="$(CGO_CFLAGS)" $(GO) build $(GOFLAGS) $(EXTRA_GOFLAGS) -tags '$(TAGS)' -ldflags '-s -w $(LDFLAGS)' -o $@
+	CGO_CFLAGS="$(CGO_CFLAGS)" $(GO) build $(GOFLAGS) $(EXTRA_GOFLAGS) -tags 'netgo osusergo $(TAGS)' -ldflags '-s -w -linkmode external -extldflags "-static" $(LDFLAGS)' -o $@
 
 .PHONY: release
 release: frontend generate release-linux release-copy release-compress vendor release-sources release-check