From bb9200860eea552a18b44639b44db6a6b8d2a72e Mon Sep 17 00:00:00 2001 From: Richie Cahill Date: Sat, 6 Dec 2025 17:31:08 -0500 Subject: [PATCH] adding fix_eval_warnings.yml --- .github/workflows/fix_eval_warnings.yml | 47 +++++++++ .gitignore | 1 + python/tools/fix_eval_warnings.py | 127 ++++++++++++++++++++++++ 3 files changed, 175 insertions(+) create mode 100644 .github/workflows/fix_eval_warnings.yml create mode 100755 python/tools/fix_eval_warnings.py diff --git a/.github/workflows/fix_eval_warnings.yml b/.github/workflows/fix_eval_warnings.yml new file mode 100644 index 0000000..0af067d --- /dev/null +++ b/.github/workflows/fix_eval_warnings.yml @@ -0,0 +1,47 @@ +name: Fix Evaluation Warnings + +on: + workflow_run: + workflows: ["build_systems"] + types: + - completed + +permissions: + contents: write + pull-requests: write + actions: read + +jobs: + analyze-and-fix: + runs-on: self-hosted + if: ${{ github.event.workflow_run.conclusion == 'success' || github.event.workflow_run.conclusion == 'failure' }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Download logs + env: + GH_TOKEN: ${{ github.token }} + RUN_ID: ${{ github.event.workflow_run.id }} + run: | + gh run view $RUN_ID --log > build.log + + - name: Run Fix Script + env: + GITHUB_TOKEN: ${{ github.token }} + GITHUB_REPOSITORY: ${{ github.repository }} + RUN_ID: ${{ github.event.workflow_run.id }} + run: | + python3 python/tools/fix_eval_warnings.py build.log + + - name: Create Pull Request + if: hashFiles('fix_suggestions.md') != '' + uses: peter-evans/create-pull-request@v6 + with: + token: ${{ github.token }} + commit-message: "fix: automated evaluation warning fixes" + title: "fix: automated evaluation warning fixes" + body-path: fix_suggestions.md + branch: "auto-fix-eval-warnings-${{ github.event.workflow_run.id }}" + base: main + labels: "automated-fix" diff --git a/.gitignore b/.gitignore index 24128cd..5a3f9cd 100644 --- a/.gitignore +++ b/.gitignore @@ -165,3 +165,4 @@ test.* # syncthing .stfolder +fix_suggestions.md diff --git a/python/tools/fix_eval_warnings.py b/python/tools/fix_eval_warnings.py new file mode 100755 index 0000000..7231388 --- /dev/null +++ b/python/tools/fix_eval_warnings.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 +""" +Script to detect "evaluation warning:" in logs and suggest fixes using GitHub Models. +""" +import os +import sys +import re +import requests +import json +from pathlib import Path + +# Configuration +GITHUB_TOKEN = os.environ.get("GITHUB_TOKEN") +GITHUB_REPOSITORY = os.environ.get("GITHUB_REPOSITORY") +PR_NUMBER = os.environ.get("PR_NUMBER") # If triggered by PR +RUN_ID = os.environ.get("RUN_ID") + +# GitHub Models API Endpoint (OpenAI compatible) +# https://github.com/marketplace/models +API_BASE = "https://models.inference.ai.azure.com" +# Default to gpt-4o, but allow override via env var +MODEL_NAME = os.environ.get("MODEL_NAME", "gpt-4o") + +def get_log_content(run_id): + """Fetches the logs for a specific workflow run.""" + print(f"Fetching logs for run ID: {run_id}") + headers = { + "Authorization": f"Bearer {GITHUB_TOKEN}", + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28" + } + + # List artifacts to find logs (or use jobs API) + # For simplicity, we might need to use 'gh' cli in the workflow to download logs + # But let's try to read from a file if passed as argument, which is easier for the workflow + return None + +def parse_warnings(log_file_path): + """Parses the log file for evaluation warnings.""" + warnings = [] + with open(log_file_path, 'r', encoding='utf-8', errors='ignore') as f: + for line in f: + if "evaluation warning:" in line: + warnings.append(line.strip()) + return warnings + +def generate_fix(warning_msg): + """Calls GitHub Models to generate a fix for the warning.""" + print(f"Generating fix for: {warning_msg}") + + prompt = f""" + I encountered the following Nix evaluation warning: + + `{warning_msg}` + + Please explain what this warning means and suggest how to fix it in the Nix code. + If possible, provide the exact code change in a diff format or a clear description of what to change. + """ + + headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {GITHUB_TOKEN}" + } + + payload = { + "messages": [ + { + "role": "system", + "content": "You are an expert NixOS and Nix language developer." + }, + { + "role": "user", + "content": prompt + } + ], + "model": MODEL_NAME, + "temperature": 0.1 + } + + try: + response = requests.post( + f"{API_BASE}/chat/completions", + headers=headers, + json=payload + ) + response.raise_for_status() + result = response.json() + return result['choices'][0]['message']['content'] + except Exception as e: + print(f"Error calling LLM: {e}") + return None + +def main(): + if len(sys.argv) < 2: + print("Usage: fix_eval_warnings.py ") + sys.exit(1) + + log_file = sys.argv[1] + if not os.path.exists(log_file): + print(f"Log file not found: {log_file}") + sys.exit(1) + + warnings = parse_warnings(log_file) + if not warnings: + print("No evaluation warnings found.") + sys.exit(0) + + print(f"Found {len(warnings)} warnings.") + + # Process unique warnings to save tokens + unique_warnings = list(set(warnings)) + + fixes = [] + for warning in unique_warnings: + fix = generate_fix(warning) + if fix: + fixes.append(f"## Warning\n`{warning}`\n\n## Suggested Fix\n{fix}\n") + + # Output fixes to a markdown file for the PR body + with open("fix_suggestions.md", "w") as f: + f.write("# Automated Fix Suggestions\n\n") + f.write("\n---\n".join(fixes)) + + print("Fix suggestions written to fix_suggestions.md") + +if __name__ == "__main__": + main()