From 9d903fbb43406949aa5d7c479520dbcd7fdd963f Mon Sep 17 00:00:00 2001 From: test Date: Tue, 25 Aug 2026 11:45:05 +0000 Subject: [PATCH] Add incremental Forgejo Actions build workflow --- .forgejo/workflows/build-rpms.yml | 122 +++++++++++++++ ci/build-plan.py | 245 ++++++++++++++++++++++++++++++ ci/build-tiered.sh | 97 ++++++++++++ 3 files changed, 464 insertions(+) create mode 100644 .forgejo/workflows/build-rpms.yml create mode 100755 ci/build-plan.py create mode 100755 ci/build-tiered.sh diff --git a/.forgejo/workflows/build-rpms.yml b/.forgejo/workflows/build-rpms.yml new file mode 100644 index 0000000..d3c9acf --- /dev/null +++ b/.forgejo/workflows/build-rpms.yml @@ -0,0 +1,122 @@ +name: Build RPMs + +on: + push: + paths: + - "**.spec" + - "build-order.txt" + - "ci/**" + - ".forgejo/workflows/**" + workflow_dispatch: + +jobs: + build: + runs-on: almalinux-10 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Install build tools + run: | + dnf install -y \ + rpm-build \ + rpmdevtools \ + mock \ + curl + + - name: Compute build plan + id: plan + run: | + if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + echo "Mode: manual trigger, full rebuild" + python3 ci/build-plan.py > /tmp/build-plan.json + elif [ "${{ github.event.before }}" = "0000000000000000000000000000000000000000" ]; then + echo "Mode: new branch, full rebuild" + python3 ci/build-plan.py > /tmp/build-plan.json + else + echo "Mode: incremental (diff ${{ github.event.before }}..${{ github.sha }})" + python3 ci/build-plan.py "${{ github.event.before }}" "${{ github.sha }}" > /tmp/build-plan.json + fi + + cat /tmp/build-plan.json | python3 -m json.tool + TOTAL=$(python3 -c "import json; print(json.load(open('/tmp/build-plan.json'))['total'])") + echo "total=${TOTAL}" >> "$GITHUB_OUTPUT" + + if [ "$TOTAL" = "0" ]; then + echo "Nothing to build, exiting" + exit 0 + fi + + - name: Add runner user to mock group + if: steps.plan.outputs.total != '0' + run: usermod -aG mock $(whoami) || true + + - name: Setup RPM build tree + if: steps.plan.outputs.total != '0' + run: rpmdev-setuptree + + - name: Configure mock chroot + if: steps.plan.outputs.total != '0' + run: | + cat > /etc/mock/sonicde.cfg <<'MOCKEOF' + include('/etc/mock/alma+epel-10-x86_64.cfg') + config_opts['root'] = 'sonicde' + config_opts['yum.conf'] += """ + [sonicde-rpm] + name=SonicDE RPM + baseurl=${{ github.server_url }}/api/packages/${{ github.repository_owner }}/rpm + enabled=1 + gpgcheck=0 + + [xlibre-xserver] + name=Copr xlibre-xserver + baseurl=https://download.copr.fedorainfracloud.org/results/@xlibre/xlibre-xserver/rhel+epel-10-$basearch/ + type=rpm-md + skip_if_unavailable=True + gpgcheck=1 + gpgkey=https://download.copr.fedorainfracloud.org/results/@xlibre/xlibre-xserver/pubkey.gpg + repo_gpgcheck=0 + enabled=1 + """ + MOCKEOF + + - name: Initialize mock chroot + if: steps.plan.outputs.total != '0' + run: | + mock --root sonicde --scrub=chroot || true + mock --root sonicde --init + mock --root sonicde --chroot 'groupadd -g 135 mock 2>/dev/null || true; useradd -u 135 -g 135 -d /builddir -s /bin/bash mockbuild 2>/dev/null || true' + + - name: Build packages tier by tier + if: steps.plan.outputs.total != '0' + run: | + echo "Building ${{ steps.plan.outputs.total }} packages..." + bash ci/build-tiered.sh /tmp/build-plan.json + + - name: Upload RPMs to Forgejo Package Registry + if: always() && steps.plan.outputs.total != '0' + run: | + FORGEJO_URL="${{ github.server_url }}" + OWNER="${{ github.repository_owner }}" + TOKEN="${{ secrets.PACKAGE_TOKEN }}" + + if [ -z "$TOKEN" ]; then + echo "PACKAGE_TOKEN secret not set, skipping upload" + exit 0 + fi + + find ~/mock-results -name "*.rpm" ! -name "*.src.rpm" | while read rpm; do + FILENAME=$(basename "$rpm") + echo "Uploading $FILENAME ..." + curl --fail-with-body \ + --user "${OWNER}:${TOKEN}" \ + --upload-file "$rpm" \ + "${FORGEJO_URL}/api/packages/${OWNER}/rpm/upload" || echo " upload failed for $FILENAME" + done + + - name: Clean up + if: always() + run: | + rm -rf ~/mock-results ~/rpmbuild/SRPMS/* diff --git a/ci/build-plan.py b/ci/build-plan.py new file mode 100755 index 0000000..5716f15 --- /dev/null +++ b/ci/build-plan.py @@ -0,0 +1,245 @@ +#!/usr/bin/env python3 +"""Compute which packages to build. + +Three modes: + ci/build-plan.py # full rebuild (all packages) + ci/build-plan.py # incremental: diff base..head + ci/build-plan.py --missing # build packages not in Forgejo repo + +Outputs JSON: {"tiers": [...], "changed": [...], "total": N} +""" +import glob +import json +import os +import re +import subprocess +import sys +import urllib.request +from collections import defaultdict + +REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +ORDER_FILE = os.path.join(REPO, "build-order.txt") + + +def parse_build_order(): + tiers = [] + current_tier = None + current_pkgs = [] + with open(ORDER_FILE) as f: + for line in f: + line = line.strip() + m = re.match(r"^\[tier(\d+)\]$", line) + if m: + if current_tier is not None: + tiers.append((current_tier, current_pkgs)) + current_tier = int(m.group(1)) + current_pkgs = [] + elif line and not line.startswith("#"): + current_pkgs.append(line) + if current_tier is not None: + tiers.append((current_tier, current_pkgs)) + return tiers + + +def build_dependency_graph(): + specs = sorted(glob.glob(os.path.join(REPO, "*", "*.spec"))) + specs = [s for s in specs + if os.path.basename(s)[:-5] == os.path.basename(os.path.dirname(s))] + + provides = {} + brs = defaultdict(set) + + for spec in specs: + pkg = os.path.basename(spec)[:-5] + out = subprocess.run( + ["rpmspec", "-P", spec], capture_output=True, text=True + ).stdout + if not out: + out = open(spec, errors="replace").read() + names = {pkg} + in_files = False + for line in out.splitlines(): + s = line.strip() + m = re.match(r"%package\s+(?:-n\s+(\S+)|(\S+))", s) + if m: + names.add(m.group(1) or f"{pkg}-{m.group(2)}") + continue + if s.startswith("%files"): + in_files = True + elif s.startswith("%") and re.match( + r"%(prep|build|install|conf|check|changelog|description|post|pre)", s + ): + in_files = s.startswith("%files") + m = re.match(r"(Provides|BuildRequires)\s*:\s*(.+)", s) + if m: + for dep in re.split(r"\s+(?=[A-Za-z_%(/])", m.group(2).strip()): + cap = dep.split()[0].replace("%{?_isa}", "").strip() + if not cap or cap.startswith(("%", "(")): + continue + if m.group(1) == "Provides": + names.add(cap) + else: + brs[pkg].add(cap) + if in_files: + for cm in re.findall(r"/cmake/([A-Za-z0-9_.+-]+)/?", s): + names.add(f"cmake({cm})") + for pc in re.findall(r"/([A-Za-z0-9_.+-]+)\.pc\b", s): + names.add(f"pkgconfig({pc})") + for n in names: + provides.setdefault(n, pkg) + + forward = defaultdict(set) + for pkg, caps in brs.items(): + for cap in caps: + owner = provides.get(cap) + if owner and owner != pkg: + forward[pkg].add(owner) + + return forward + + +def compute_affected(changed_pkgs, forward_graph): + reverse = defaultdict(set) + for pkg, deps in forward_graph.items(): + for dep in deps: + reverse[dep].add(pkg) + + affected = set(changed_pkgs) + queue = list(changed_pkgs) + while queue: + pkg = queue.pop(0) + for dependent in reverse.get(pkg, ()): + if dependent not in affected: + affected.add(dependent) + queue.append(dependent) + return affected + + +def get_spec_version(pkg): + """Return (version, release) for a package from its spec.""" + spec = os.path.join(REPO, pkg, f"{pkg}.spec") + out = subprocess.run( + ["rpmspec", "-q", "--srpm", + "--queryformat", "%{VERSION} %{RELEASE}", spec], + capture_output=True, text=True + ).stdout.strip() + if " " in out: + return out.split(" ", 1) + return None, None + + +def query_forgejo_packages(): + """Query the Forgejo RPM registry for existing packages. + + Returns a dict: {package_name: {(version, release)}} for all subpackages. + """ + forgejo_url = os.environ.get("FORGEJO_URL", "") + owner = os.environ.get("FORGEJO_OWNER", "") + if not forgejo_url or not owner: + return {} + + packages = {} + page = 1 + while True: + url = f"{forgejo_url}/api/packages/{owner}/rpm?page={page}&limit=50" + try: + req = urllib.request.Request(url) + with urllib.request.urlopen(req, timeout=30) as resp: + data = json.loads(resp.read()) + except Exception: + break + if not data: + break + for entry in data: + name = entry.get("name", "") + if not name: + continue + # Strip arch suffix and subpackage qualifiers to map back to source pkg + ver = entry.get("version", "") + rel = "" + # Forgejo RPM packages store version-release in the version field + if "-" in ver: + ver, rel = ver.rsplit("-", 1) + packages.setdefault(name, set()).add((ver, rel)) + page += 1 + return packages + + +def compute_missing(all_pkgs, tiers, forgejo_packages): + """Find packages whose spec version is not in the Forgejo registry.""" + if not forgejo_packages: + return set(all_pkgs) + + missing = set() + for pkg in all_pkgs: + ver, rel = get_spec_version(pkg) + if ver is None: + continue + # Check if the main package or any subpackage with this version exists + found = False + for name, versions in forgejo_packages.items(): + # Match either the exact package name or a subpackage (pkg-foo) + if name == pkg or name.startswith(f"{pkg}-"): + for (fv, fr) in versions: + if fv == ver: + found = True + break + if found: + break + if not found: + missing.add(pkg) + return missing + + +def main(): + tiers = parse_build_order() + all_pkgs = [pkg for _, pkgs in tiers for pkg in pkgs] + + # Determine mode + if len(sys.argv) >= 3 and sys.argv[1] != "--missing": + # Incremental: diff base..head + base, head = sys.argv[1], sys.argv[2] + changed_files = subprocess.run( + ["git", "diff", "--name-only", base, head], + capture_output=True, text=True, cwd=REPO, + ).stdout.split() + changed_pkgs = { + os.path.basename(f).replace(".spec", "") + for f in changed_files if f.endswith(".spec") + } + for f in changed_files: + if "/" in f and not f.endswith(".spec"): + pkg = f.split("/")[0] + if pkg in all_pkgs: + changed_pkgs.add(pkg) + elif len(sys.argv) >= 2 and sys.argv[1] == "--missing": + # Missing mode: build packages not in Forgejo registry + forgejo_pkgs = query_forgejo_packages() + changed_pkgs = compute_missing(all_pkgs, tiers, forgejo_pkgs) + else: + # Full rebuild + changed_pkgs = set(all_pkgs) + + if not changed_pkgs: + print(json.dumps({"tiers": [], "changed": [], "total": 0})) + return + + forward = build_dependency_graph() + affected = compute_affected(changed_pkgs, forward) + + plan = [] + for tier_num, pkgs in tiers: + tier_affected = [p for p in pkgs if p in affected] + if tier_affected: + plan.append({"tier": tier_num, "packages": sorted(tier_affected)}) + + result = { + "tiers": plan, + "changed": sorted(changed_pkgs), + "total": sum(len(t["packages"]) for t in plan), + } + print(json.dumps(result)) + + +if __name__ == "__main__": + main() diff --git a/ci/build-tiered.sh b/ci/build-tiered.sh new file mode 100755 index 0000000..9e3ae05 --- /dev/null +++ b/ci/build-tiered.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash +# Build SonicDE packages in tier order inside a mock chroot. +# +# Usage: ci/build-tiered.sh +# +# After each package is built, its RPMs are installed into the mock chroot +# so they're available as build dependencies for subsequent tiers. +set -euo pipefail + +PLAN_FILE="${1:?usage: build-tiered.sh }" +REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +SOURCES_DIR="${HOME}/rpmbuild/SOURCES" +RESULTS_DIR="${HOME}/mock-results" +MOCK_ROOT="${MOCK_ROOT:-sonicde}" + +mkdir -p "$RESULTS_DIR" "$SOURCES_DIR" + +TOTAL=$(python3 -c "import json; print(json.load(open('$PLAN_FILE'))['total'])") +BUILT=0 +FAILED=() + +NUM_TIERS=$(python3 -c "import json; print(len(json.load(open('$PLAN_FILE'))['tiers']))") + +for tier_idx in $(seq 0 $((NUM_TIERS - 1))); do + TIER_NUM=$(python3 -c "import json; print(json.load(open('$PLAN_FILE'))['tiers'][$tier_idx]['tier'])") + PACKAGES=$(python3 -c "import json; print('\n'.join(json.load(open('$PLAN_FILE'))['tiers'][$tier_idx]['packages']))") + PKG_COUNT=$(echo "$PACKAGES" | wc -l) + + echo "" + echo "========================================" + echo " Tier ${TIER_NUM}: ${PKG_COUNT} packages" + echo "========================================" + + for pkg in $PACKAGES; do + BUILT=$((BUILT + 1)) + echo "" + echo "[${BUILT}/${TOTAL}] Building ${pkg}..." + + spec_dir="${REPO_DIR}/${pkg}" + spec="${spec_dir}/${pkg}.spec" + + if [ ! -f "$spec" ]; then + echo " SKIP: spec not found: ${spec}" + continue + fi + + # Copy local source files (patches, configs, etc.) + find "$spec_dir" -maxdepth 1 -type f ! -name '*.spec' \ + -exec cp -p {} "$SOURCES_DIR/" \; + + # Download URL sources + spectool -g -C "$SOURCES_DIR" "$spec" 2>&1 | tail -2 || true + + # Build SRPM + rpmbuild -bs \ + --define "_topdir ${HOME}/rpmbuild" \ + --define "_disable_source_fetch 0" \ + "$spec" 2>&1 | tail -3 + + srpm_name=$(rpmspec -q --srpm \ + --queryformat '%{NAME}-%{VERSION}-%{RELEASE}.src.rpm' "$spec") + srpm_path="${HOME}/rpmbuild/SRPMS/${srpm_name}" + + # Build with mock + pkg_result_dir="${RESULTS_DIR}/${pkg}" + rm -rf "$pkg_result_dir" + mkdir -p "$pkg_result_dir" + + if mock --root "$MOCK_ROOT" \ + --resultdir "$pkg_result_dir" \ + --no-clean --rebuild "$srpm_path" 2>&1 | tail -10; then + echo " OK: ${pkg}" + + # Install the built RPMs into the mock chroot so later tiers + # can use them as build dependencies. + echo " Installing ${pkg} RPMs into chroot..." + mock --root "$MOCK_ROOT" --no-clean --install \ + $(find "$pkg_result_dir" -name '*.rpm' ! -name '*.src.rpm') \ + 2>&1 | tail -3 || echo " WARNING: could not install ${pkg} into chroot" + else + echo " FAILED: ${pkg}" + FAILED+=("$pkg") + fi + + # Clean SRPM to save disk space + rm -f "$srpm_path" + done +done + +echo "" +echo "========================================" +echo " Summary: $((TOTAL - ${#FAILED[@]}))/${TOTAL} succeeded" +if [ ${#FAILED[@]} -gt 0 ]; then + echo " Failed: ${FAILED[*]}" + exit 1 +fi +echo "========================================"