diff --git a/.gitea/workflows/build.yml b/.gitea/workflows/build.yml index d74fb89..9e09dc2 100644 --- a/.gitea/workflows/build.yml +++ b/.gitea/workflows/build.yml @@ -1,5 +1,454 @@ -on: [push, pull_request, workflow_dispatch] +name: Build + +on: + push: + paths: + - ".github/workflows/build.yml" + - "app/**" + pull_request: + paths: + - ".github/workflows/build.yml" + - "app/**" + schedule: + - cron: "22 4 * * *" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name == 'schedule' }} + cancel-in-progress: true + +permissions: {} jobs: build: - uses: zmkfirmware/zmk/.github/workflows/build-user-config.yml@main + if: ${{ always() }} + runs-on: ubuntu-latest + container: + image: docker.io/zmkfirmware/zmk-build-arm:3.5 + needs: compile-matrix + strategy: + matrix: + include: ${{ fromJSON(needs.compile-matrix.outputs.include-list) }} + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Cache west modules + uses: actions/cache@v4 + env: + cache-name: cache-zephyr-modules + with: + path: | + modules/ + tools/ + zephyr/ + bootloader/ + key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('app/west.yml') }} + restore-keys: | + ${{ runner.os }}-build-${{ env.cache-name }}- + ${{ runner.os }}-build- + ${{ runner.os }}- + timeout-minutes: 2 + continue-on-error: true + - name: Initialize workspace (west init) + run: west init -l app + - name: Update modules (west update) + run: west update + - name: Export Zephyr CMake package (west zephyr-export) + run: west zephyr-export + - name: Use Node.js + uses: actions/setup-node@v4 + with: + node-version: "14.x" + - name: Install @actions/artifact + run: npm install @actions/artifact + - name: Build + uses: actions/github-script@v7 + id: boards-list + with: + script: | + const fs = require('fs'); + + const execSync = require('child_process').execSync; + + const buildShieldArgs = JSON.parse(`${{ matrix.shieldArgs }}`); + + let error = false; + + for (const shieldArgs of buildShieldArgs) { + try { + console.log(`::group::${{ matrix.board}} ${shieldArgs.shield} Build`) + + const output = execSync(`west build -s app -p -b ${{ matrix.board }} -- ${shieldArgs.shield ? '-DSHIELD="' + shieldArgs.shield + '"' : ''} ${shieldArgs['cmake-args'] || ''}`); + + console.log(output.toString()); + } catch (e) { + console.error(`::error::Failed to build ${{ matrix.board }} ${shieldArgs.shield} ${shieldArgs['cmake-args']}`); + console.error(e); + error = true; + } finally { + console.log('::endgroup::'); + } + } + + if (error) { + throw new Error('Failed to build one or more configurations'); + } + - name: Upload artifacts + uses: actions/github-script@v7 + continue-on-error: ${{ github.event_name == 'pull_request' }} + id: boards-upload + with: + script: | + const fs = require('fs'); + const {default: artifact} = require('@actions/artifact'); + + const buildShieldArgs = JSON.parse(`${{ matrix.shieldArgs }}`); + + let error = false; + + for (const shieldArgs of buildShieldArgs) { + try { + console.log(`::group::${{ matrix.board}} ${shieldArgs.shield} Upload`) + + const fileExtensions = ["hex", "uf2"]; + + const files = fileExtensions + .map(extension => "build/zephyr/zmk." + extension) + .filter(path => fs.existsSync(path)); + + const rootDirectory = 'build/zephyr'; + const options = { + continueOnError: true + } + + const cmakeName = shieldArgs['cmake-args'] ? '-' + (shieldArgs.nickname || shieldArgs['cmake-args'].split(' ').join('')) : ''; + const artifactName = `${{ matrix.board }}${shieldArgs.shield ? '-' + shieldArgs.shield : ''}${cmakeName}-zmk`; + + await artifact.uploadArtifact(artifactName, files, rootDirectory, options); + } catch (e) { + console.error(`::error::Failed to upload ${{ matrix.board }} ${shieldArgs.shield} ${shieldArgs['cmake-args']}`); + console.error(e); + error = true; + } finally { + console.log('::endgroup::'); + } + } + + if (error) { + throw new Error('Failed to build one or more configurations'); + } + compile-matrix: + if: ${{ !cancelled() }} + runs-on: ubuntu-latest + needs: [core-coverage, board-changes, nightly] + outputs: + include-list: ${{ steps.compile-list.outputs.result }} + steps: + - name: Join build lists + uses: actions/github-script@v7 + id: compile-list + with: + script: | + const coreCoverage = `${{ needs.core-coverage.outputs.core-include }}` || "[]"; + const boardChanges = `${{ needs.board-changes.outputs.boards-include }}` || "[]"; + const nightly = `${{ needs.nightly.outputs.nightly-include }}` || "[]"; + + const combined = [ + ...JSON.parse(coreCoverage), + ...JSON.parse(boardChanges), + ...JSON.parse(nightly) + ]; + const combinedUnique = [...new Map(combined.map(el => [JSON.stringify(el), el])).values()]; + + const perBoard = {}; + + for (const configuration of combinedUnique) { + if (!perBoard[configuration.board]) + perBoard[configuration.board] = []; + + perBoard[configuration.board].push({ + shield: configuration.shield, + 'cmake-args': configuration['cmake-args'], + nickname: configuration.nickname + }) + } + + return Object.entries(perBoard).map(([board, shieldArgs]) => ({ + board, + shieldArgs: JSON.stringify(shieldArgs), + })); + core-coverage: + if: ${{ needs.get-changed-files.outputs.core-changes == 'true' }} + runs-on: ubuntu-latest + needs: get-changed-files + outputs: + core-include: ${{ steps.core-list.outputs.result }} + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Use Node.js + uses: actions/setup-node@v4 + with: + node-version: "14.x" + - name: Install js-yaml + run: npm install js-yaml + - uses: actions/github-script@v7 + id: core-list + with: + script: | + const fs = require('fs'); + const yaml = require('js-yaml'); + + const coreCoverage = yaml.load(fs.readFileSync('app/core-coverage.yml', 'utf8')); + + let include = coreCoverage.board.flatMap(board => + coreCoverage.shield.map(shield => ({ board, shield })) + ); + + return [...include, ...coreCoverage.include]; + board-changes: + if: ${{ needs.get-changed-files.outputs.board-changes == 'true' }} + runs-on: ubuntu-latest + needs: [get-grouped-hardware, get-changed-files] + outputs: + boards-include: ${{ steps.boards-list.outputs.result }} + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Use Node.js + uses: actions/setup-node@v4 + with: + node-version: "14.x" + - name: Install js-yaml + run: npm install js-yaml + - uses: actions/github-script@v7 + id: boards-list + with: + script: | + const fs = require('fs'); + const yaml = require('js-yaml'); + + const changedFiles = JSON.parse(`${{ needs.get-changed-files.outputs.changed-files }}`); + const metadata = JSON.parse(`${{ needs.get-grouped-hardware.outputs.organized-metadata }}`); + const boardChanges = new Set(changedFiles.filter(f => f.startsWith('app/boards')).map(f => f.split('/').slice(0, 4).join('/'))); + + return (await Promise.all([...boardChanges].flatMap(async bc => { + const globber = await glob.create(bc + "/*.zmk.yml"); + const files = await globber.glob(); + + const aggregated = files.flatMap((f) => + yaml.loadAll(fs.readFileSync(f, "utf8")) + ); + + const boardAndShield = (b, s) => { + if (s.siblings) { + return s.siblings.map(shield => ({ + board: b.id, + shield, + })); + } else { + return { + board: b.id, + shield: s.id + }; + } + } + + return aggregated.flatMap(hm => { + switch (hm.type) { + case "board": + if (hm.features && hm.features.includes("keys")) { + if (hm.siblings) { + return hm.siblings.map(board => ({ + board, + })); + } else { + return { + board: hm.id + }; + } + } else if (hm.exposes) { + return hm.exposes.flatMap(i => + metadata.interconnects[i].shields.flatMap(s => boardAndShield(hm, s)) + ); + } else { + console.error("Board without keys or interconnect"); + } + break; + case "shield": + if (hm.features && hm.features.includes("keys")) { + return hm.requires.flatMap(i => + metadata.interconnects[i].boards.flatMap(b => boardAndShield(b, hm)) + ); + } else { + console.warn("Unhandled shield without keys"); + return []; + } + break; + case "interconnect": + return []; + } + }); + }))).flat(); + nightly: + if: ${{ github.event_name == 'schedule' && github.repository_owner == 'zmkfirmware' }} + runs-on: ubuntu-latest + needs: get-grouped-hardware + outputs: + nightly-include: ${{ steps.nightly-list.outputs.result }} + steps: + - name: Create nightly list + uses: actions/github-script@v7 + id: nightly-list + with: + script: | + const metadata = JSON.parse(`${{ needs.get-grouped-hardware.outputs.organized-metadata }}`); + + let includeOnboard = metadata.onboard.flatMap(b => { + if (b.siblings) { + return b.siblings.map(board => ({ + board, + })); + } else { + return { + board: b.id, + }; + } + }); + + let includeInterconnect = Object.values(metadata.interconnects).flatMap(i => + i.boards.flatMap(b => + i.shields.flatMap(s => { + if (s.siblings) { + return s.siblings.map(shield => ({ + board: b.id, + shield, + })); + } else { + return { + board: b.id, + shield: s.id, + }; + } + }) + ) + ); + + return [...includeOnboard, ...includeInterconnect]; + get-grouped-hardware: + runs-on: ubuntu-latest + outputs: + organized-metadata: ${{ steps.organize-metadata.outputs.result }} + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Use Node.js + uses: actions/setup-node@v4 + with: + node-version: "14.x" + - name: Install js-yaml + run: npm install js-yaml + - name: Aggregate Metadata + uses: actions/github-script@v7 + id: aggregate-metadata + with: + script: | + const fs = require('fs'); + const yaml = require('js-yaml'); + + const globber = await glob.create("app/boards/**/*.zmk.yml"); + const files = await globber.glob(); + + const aggregated = files.flatMap((f) => + yaml.loadAll(fs.readFileSync(f, "utf8")) + ); + + return JSON.stringify(aggregated).replace(/\\/g,"\\\\").replace(/`/g,"\\`"); + result-encoding: string + + - name: Organize Metadata + uses: actions/github-script@v7 + id: organize-metadata + with: + script: | + const hardware = JSON.parse(`${{ steps.aggregate-metadata.outputs.result }}`); + + const grouped = hardware.reduce((agg, hm) => { + switch (hm.type) { + case "board": + if (hm.features && hm.features.includes("keys")) { + agg.onboard.push(hm); + } else if (hm.exposes) { + hm.exposes.forEach((element) => { + let ic = agg.interconnects[element] || { + boards: [], + shields: [], + }; + ic.boards.push(hm); + agg.interconnects[element] = ic; + }); + } else { + console.error("Board without keys or interconnect"); + } + break; + case "shield": + if (hm.features && hm.features.includes("keys")) { + hm.requires.forEach((id) => { + let ic = agg.interconnects[id] || { boards: [], shields: [] }; + ic.shields.push(hm); + agg.interconnects[id] = ic; + }); + } + break; + case "interconnect": + let ic = agg.interconnects[hm.id] || { boards: [], shields: [] }; + ic.interconnect = hm; + agg.interconnects[hm.id] = ic; + break; + } + return agg; + }, + { onboard: [], interconnects: {} }); + + return JSON.stringify(grouped).replace(/\\/g,"\\\\").replace(/`/g,"\\`"); + result-encoding: string + get-changed-files: + if: ${{ github.event_name != 'schedule' }} + runs-on: ubuntu-latest + outputs: + changed-files: ${{ steps.changed-files.outputs.all_changed_files }} + board-changes: ${{ steps.board-changes.outputs.result }} + core-changes: ${{ steps.core-changes.outputs.result }} + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: tj-actions/changed-files@v44 + id: changed-files + with: + json: true + escape_json: false + - uses: actions/github-script@v7 + id: board-changes + with: + script: | + const changedFiles = JSON.parse(`${{ steps.changed-files.outputs.all_changed_files }}`); + const boardChanges = changedFiles.filter(f => f.startsWith('app/boards')); + return boardChanges.length ? 'true' : 'false'; + result-encoding: string + - uses: actions/github-script@v7 + id: core-changes + with: + script: | + const changedFiles = JSON.parse(`${{ steps.changed-files.outputs.all_changed_files }}`); + const boardChanges = changedFiles.filter(f => f.startsWith('app/boards')); + const appChanges = changedFiles.filter(f => f.startsWith('app')); + const ymlChanges = changedFiles.includes('.github/workflows/build.yml'); + return boardChanges.length < appChanges.length || ymlChanges ? 'true' : 'false'; + result-encoding: string