Add incremental Forgejo Actions build workflow

.forgejo/workflows/build-rpms.yml

@@ -0,0 +1,122 @@
+name: Build RPMs
+
+on:
+  push:
+    paths:
+      - "**.spec"
+      - "build-order.txt"
+      - "ci/**"
+      - ".forgejo/workflows/**"
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: almalinux-10
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Install build tools
+        run: |
+          dnf install -y \
+            rpm-build \
+            rpmdevtools \
+            mock \
+            curl
+
+      - name: Compute build plan
+        id: plan
+        run: |
+          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+            echo "Mode: manual trigger, full rebuild"
+            python3 ci/build-plan.py > /tmp/build-plan.json
+          elif [ "${{ github.event.before }}" = "0000000000000000000000000000000000000000" ]; then
+            echo "Mode: new branch, full rebuild"
+            python3 ci/build-plan.py > /tmp/build-plan.json
+          else
+            echo "Mode: incremental (diff ${{ github.event.before }}..${{ github.sha }})"
+            python3 ci/build-plan.py "${{ github.event.before }}" "${{ github.sha }}" > /tmp/build-plan.json
+          fi
+
+          cat /tmp/build-plan.json | python3 -m json.tool
+          TOTAL=$(python3 -c "import json; print(json.load(open('/tmp/build-plan.json'))['total'])")
+          echo "total=${TOTAL}" >> "$GITHUB_OUTPUT"
+
+          if [ "$TOTAL" = "0" ]; then
+            echo "Nothing to build, exiting"
+            exit 0
+          fi
+
+      - name: Add runner user to mock group
+        if: steps.plan.outputs.total != '0'
+        run: usermod -aG mock $(whoami) || true
+
+      - name: Setup RPM build tree
+        if: steps.plan.outputs.total != '0'
+        run: rpmdev-setuptree
+
+      - name: Configure mock chroot
+        if: steps.plan.outputs.total != '0'
+        run: |
+          cat > /etc/mock/sonicde.cfg <<'MOCKEOF'
+          include('/etc/mock/alma+epel-10-x86_64.cfg')
+          config_opts['root'] = 'sonicde'
+          config_opts['yum.conf'] += """
+          [sonicde-rpm]
+          name=SonicDE RPM
+          baseurl=${{ github.server_url }}/api/packages/${{ github.repository_owner }}/rpm
+          enabled=1
+          gpgcheck=0
+
+          [xlibre-xserver]
+          name=Copr xlibre-xserver
+          baseurl=https://download.copr.fedorainfracloud.org/results/@xlibre/xlibre-xserver/rhel+epel-10-$basearch/
+          type=rpm-md
+          skip_if_unavailable=True
+          gpgcheck=1
+          gpgkey=https://download.copr.fedorainfracloud.org/results/@xlibre/xlibre-xserver/pubkey.gpg
+          repo_gpgcheck=0
+          enabled=1
+          """
+          MOCKEOF
+
+      - name: Initialize mock chroot
+        if: steps.plan.outputs.total != '0'
+        run: |
+          mock --root sonicde --scrub=chroot || true
+          mock --root sonicde --init
+          mock --root sonicde --chroot 'groupadd -g 135 mock 2>/dev/null || true; useradd -u 135 -g 135 -d /builddir -s /bin/bash mockbuild 2>/dev/null || true'
+
+      - name: Build packages tier by tier
+        if: steps.plan.outputs.total != '0'
+        run: |
+          echo "Building ${{ steps.plan.outputs.total }} packages..."
+          bash ci/build-tiered.sh /tmp/build-plan.json
+
+      - name: Upload RPMs to Forgejo Package Registry
+        if: always() && steps.plan.outputs.total != '0'
+        run: |
+          FORGEJO_URL="${{ github.server_url }}"
+          OWNER="${{ github.repository_owner }}"
+          TOKEN="${{ secrets.PACKAGE_TOKEN }}"
+
+          if [ -z "$TOKEN" ]; then
+            echo "PACKAGE_TOKEN secret not set, skipping upload"
+            exit 0
+          fi
+
+          find ~/mock-results -name "*.rpm" ! -name "*.src.rpm" | while read rpm; do
+            FILENAME=$(basename "$rpm")
+            echo "Uploading $FILENAME ..."
+            curl --fail-with-body \
+              --user "${OWNER}:${TOKEN}" \
+              --upload-file "$rpm" \
+              "${FORGEJO_URL}/api/packages/${OWNER}/rpm/upload" || echo "  upload failed for $FILENAME"
+          done
+
+      - name: Clean up
+        if: always()
+        run: |
+          rm -rf ~/mock-results ~/rpmbuild/SRPMS/*

ci/build-plan.py

@@ -0,0 +1,245 @@
+#!/usr/bin/env python3
+"""Compute which packages to build.
+
+Three modes:
+  ci/build-plan.py                          # full rebuild (all packages)
+  ci/build-plan.py <base> <head>            # incremental: diff base..head
+  ci/build-plan.py --missing                # build packages not in Forgejo repo
+
+Outputs JSON: {"tiers": [...], "changed": [...], "total": N}
+"""
+import glob
+import json
+import os
+import re
+import subprocess
+import sys
+import urllib.request
+from collections import defaultdict
+
+REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+ORDER_FILE = os.path.join(REPO, "build-order.txt")
+
+
+def parse_build_order():
+    tiers = []
+    current_tier = None
+    current_pkgs = []
+    with open(ORDER_FILE) as f:
+        for line in f:
+            line = line.strip()
+            m = re.match(r"^\[tier(\d+)\]$", line)
+            if m:
+                if current_tier is not None:
+                    tiers.append((current_tier, current_pkgs))
+                current_tier = int(m.group(1))
+                current_pkgs = []
+            elif line and not line.startswith("#"):
+                current_pkgs.append(line)
+    if current_tier is not None:
+        tiers.append((current_tier, current_pkgs))
+    return tiers
+
+
+def build_dependency_graph():
+    specs = sorted(glob.glob(os.path.join(REPO, "*", "*.spec")))
+    specs = [s for s in specs
+             if os.path.basename(s)[:-5] == os.path.basename(os.path.dirname(s))]
+
+    provides = {}
+    brs = defaultdict(set)
+
+    for spec in specs:
+        pkg = os.path.basename(spec)[:-5]
+        out = subprocess.run(
+            ["rpmspec", "-P", spec], capture_output=True, text=True
+        ).stdout
+        if not out:
+            out = open(spec, errors="replace").read()
+        names = {pkg}
+        in_files = False
+        for line in out.splitlines():
+            s = line.strip()
+            m = re.match(r"%package\s+(?:-n\s+(\S+)|(\S+))", s)
+            if m:
+                names.add(m.group(1) or f"{pkg}-{m.group(2)}")
+                continue
+            if s.startswith("%files"):
+                in_files = True
+            elif s.startswith("%") and re.match(
+                r"%(prep|build|install|conf|check|changelog|description|post|pre)", s
+            ):
+                in_files = s.startswith("%files")
+            m = re.match(r"(Provides|BuildRequires)\s*:\s*(.+)", s)
+            if m:
+                for dep in re.split(r"\s+(?=[A-Za-z_%(/])", m.group(2).strip()):
+                    cap = dep.split()[0].replace("%{?_isa}", "").strip()
+                    if not cap or cap.startswith(("%", "(")):
+                        continue
+                    if m.group(1) == "Provides":
+                        names.add(cap)
+                    else:
+                        brs[pkg].add(cap)
+            if in_files:
+                for cm in re.findall(r"/cmake/([A-Za-z0-9_.+-]+)/?", s):
+                    names.add(f"cmake({cm})")
+                for pc in re.findall(r"/([A-Za-z0-9_.+-]+)\.pc\b", s):
+                    names.add(f"pkgconfig({pc})")
+        for n in names:
+            provides.setdefault(n, pkg)
+
+    forward = defaultdict(set)
+    for pkg, caps in brs.items():
+        for cap in caps:
+            owner = provides.get(cap)
+            if owner and owner != pkg:
+                forward[pkg].add(owner)
+
+    return forward
+
+
+def compute_affected(changed_pkgs, forward_graph):
+    reverse = defaultdict(set)
+    for pkg, deps in forward_graph.items():
+        for dep in deps:
+            reverse[dep].add(pkg)
+
+    affected = set(changed_pkgs)
+    queue = list(changed_pkgs)
+    while queue:
+        pkg = queue.pop(0)
+        for dependent in reverse.get(pkg, ()):
+            if dependent not in affected:
+                affected.add(dependent)
+                queue.append(dependent)
+    return affected
+
+
+def get_spec_version(pkg):
+    """Return (version, release) for a package from its spec."""
+    spec = os.path.join(REPO, pkg, f"{pkg}.spec")
+    out = subprocess.run(
+        ["rpmspec", "-q", "--srpm",
+         "--queryformat", "%{VERSION} %{RELEASE}", spec],
+        capture_output=True, text=True
+    ).stdout.strip()
+    if " " in out:
+        return out.split(" ", 1)
+    return None, None
+
+
+def query_forgejo_packages():
+    """Query the Forgejo RPM registry for existing packages.
+
+    Returns a dict: {package_name: {(version, release)}} for all subpackages.
+    """
+    forgejo_url = os.environ.get("FORGEJO_URL", "")
+    owner = os.environ.get("FORGEJO_OWNER", "")
+    if not forgejo_url or not owner:
+        return {}
+
+    packages = {}
+    page = 1
+    while True:
+        url = f"{forgejo_url}/api/packages/{owner}/rpm?page={page}&limit=50"
+        try:
+            req = urllib.request.Request(url)
+            with urllib.request.urlopen(req, timeout=30) as resp:
+                data = json.loads(resp.read())
+        except Exception:
+            break
+        if not data:
+            break
+        for entry in data:
+            name = entry.get("name", "")
+            if not name:
+                continue
+            # Strip arch suffix and subpackage qualifiers to map back to source pkg
+            ver = entry.get("version", "")
+            rel = ""
+            # Forgejo RPM packages store version-release in the version field
+            if "-" in ver:
+                ver, rel = ver.rsplit("-", 1)
+            packages.setdefault(name, set()).add((ver, rel))
+        page += 1
+    return packages
+
+
+def compute_missing(all_pkgs, tiers, forgejo_packages):
+    """Find packages whose spec version is not in the Forgejo registry."""
+    if not forgejo_packages:
+        return set(all_pkgs)
+
+    missing = set()
+    for pkg in all_pkgs:
+        ver, rel = get_spec_version(pkg)
+        if ver is None:
+            continue
+        # Check if the main package or any subpackage with this version exists
+        found = False
+        for name, versions in forgejo_packages.items():
+            # Match either the exact package name or a subpackage (pkg-foo)
+            if name == pkg or name.startswith(f"{pkg}-"):
+                for (fv, fr) in versions:
+                    if fv == ver:
+                        found = True
+                        break
+            if found:
+                break
+        if not found:
+            missing.add(pkg)
+    return missing
+
+
+def main():
+    tiers = parse_build_order()
+    all_pkgs = [pkg for _, pkgs in tiers for pkg in pkgs]
+
+    # Determine mode
+    if len(sys.argv) >= 3 and sys.argv[1] != "--missing":
+        # Incremental: diff base..head
+        base, head = sys.argv[1], sys.argv[2]
+        changed_files = subprocess.run(
+            ["git", "diff", "--name-only", base, head],
+            capture_output=True, text=True, cwd=REPO,
+        ).stdout.split()
+        changed_pkgs = {
+            os.path.basename(f).replace(".spec", "")
+            for f in changed_files if f.endswith(".spec")
+        }
+        for f in changed_files:
+            if "/" in f and not f.endswith(".spec"):
+                pkg = f.split("/")[0]
+                if pkg in all_pkgs:
+                    changed_pkgs.add(pkg)
+    elif len(sys.argv) >= 2 and sys.argv[1] == "--missing":
+        # Missing mode: build packages not in Forgejo registry
+        forgejo_pkgs = query_forgejo_packages()
+        changed_pkgs = compute_missing(all_pkgs, tiers, forgejo_pkgs)
+    else:
+        # Full rebuild
+        changed_pkgs = set(all_pkgs)
+
+    if not changed_pkgs:
+        print(json.dumps({"tiers": [], "changed": [], "total": 0}))
+        return
+
+    forward = build_dependency_graph()
+    affected = compute_affected(changed_pkgs, forward)
+
+    plan = []
+    for tier_num, pkgs in tiers:
+        tier_affected = [p for p in pkgs if p in affected]
+        if tier_affected:
+            plan.append({"tier": tier_num, "packages": sorted(tier_affected)})
+
+    result = {
+        "tiers": plan,
+        "changed": sorted(changed_pkgs),
+        "total": sum(len(t["packages"]) for t in plan),
+    }
+    print(json.dumps(result))
+
+
+if __name__ == "__main__":
+    main()

ci/build-tiered.sh

@@ -0,0 +1,97 @@
+#!/usr/bin/env bash
+# Build SonicDE packages in tier order inside a mock chroot.
+#
+# Usage: ci/build-tiered.sh <build-plan.json>
+#
+# After each package is built, its RPMs are installed into the mock chroot
+# so they're available as build dependencies for subsequent tiers.
+set -euo pipefail
+
+PLAN_FILE="${1:?usage: build-tiered.sh <plan.json>}"
+REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+SOURCES_DIR="${HOME}/rpmbuild/SOURCES"
+RESULTS_DIR="${HOME}/mock-results"
+MOCK_ROOT="${MOCK_ROOT:-sonicde}"
+
+mkdir -p "$RESULTS_DIR" "$SOURCES_DIR"
+
+TOTAL=$(python3 -c "import json; print(json.load(open('$PLAN_FILE'))['total'])")
+BUILT=0
+FAILED=()
+
+NUM_TIERS=$(python3 -c "import json; print(len(json.load(open('$PLAN_FILE'))['tiers']))")
+
+for tier_idx in $(seq 0 $((NUM_TIERS - 1))); do
+    TIER_NUM=$(python3 -c "import json; print(json.load(open('$PLAN_FILE'))['tiers'][$tier_idx]['tier'])")
+    PACKAGES=$(python3 -c "import json; print('\n'.join(json.load(open('$PLAN_FILE'))['tiers'][$tier_idx]['packages']))")
+    PKG_COUNT=$(echo "$PACKAGES" | wc -l)
+
+    echo ""
+    echo "========================================"
+    echo "  Tier ${TIER_NUM}: ${PKG_COUNT} packages"
+    echo "========================================"
+
+    for pkg in $PACKAGES; do
+        BUILT=$((BUILT + 1))
+        echo ""
+        echo "[${BUILT}/${TOTAL}] Building ${pkg}..."
+
+        spec_dir="${REPO_DIR}/${pkg}"
+        spec="${spec_dir}/${pkg}.spec"
+
+        if [ ! -f "$spec" ]; then
+            echo "  SKIP: spec not found: ${spec}"
+            continue
+        fi
+
+        # Copy local source files (patches, configs, etc.)
+        find "$spec_dir" -maxdepth 1 -type f ! -name '*.spec' \
+            -exec cp -p {} "$SOURCES_DIR/" \;
+
+        # Download URL sources
+        spectool -g -C "$SOURCES_DIR" "$spec" 2>&1 | tail -2 || true
+
+        # Build SRPM
+        rpmbuild -bs \
+            --define "_topdir ${HOME}/rpmbuild" \
+            --define "_disable_source_fetch 0" \
+            "$spec" 2>&1 | tail -3
+
+        srpm_name=$(rpmspec -q --srpm \
+            --queryformat '%{NAME}-%{VERSION}-%{RELEASE}.src.rpm' "$spec")
+        srpm_path="${HOME}/rpmbuild/SRPMS/${srpm_name}"
+
+        # Build with mock
+        pkg_result_dir="${RESULTS_DIR}/${pkg}"
+        rm -rf "$pkg_result_dir"
+        mkdir -p "$pkg_result_dir"
+
+        if mock --root "$MOCK_ROOT" \
+                --resultdir "$pkg_result_dir" \
+                --no-clean --rebuild "$srpm_path" 2>&1 | tail -10; then
+            echo "  OK: ${pkg}"
+
+            # Install the built RPMs into the mock chroot so later tiers
+            # can use them as build dependencies.
+            echo "  Installing ${pkg} RPMs into chroot..."
+            mock --root "$MOCK_ROOT" --no-clean --install \
+                $(find "$pkg_result_dir" -name '*.rpm' ! -name '*.src.rpm') \
+                2>&1 | tail -3 || echo "  WARNING: could not install ${pkg} into chroot"
+        else
+            echo "  FAILED: ${pkg}"
+            FAILED+=("$pkg")
+        fi
+
+        # Clean SRPM to save disk space
+        rm -f "$srpm_path"
+    done
+done
+
+echo ""
+echo "========================================"
+echo "  Summary: $((TOTAL - ${#FAILED[@]}))/${TOTAL} succeeded"
+if [ ${#FAILED[@]} -gt 0 ]; then
+    echo "  Failed: ${FAILED[*]}"
+    exit 1
+fi
+echo "========================================"