diff --git a/docs/assets/mastra-watchflow-rules.yaml b/docs/assets/mastra-watchflow-rules.yaml new file mode 100644 index 0000000..daa1df6 --- /dev/null +++ b/docs/assets/mastra-watchflow-rules.yaml @@ -0,0 +1,56 @@ +rules: + - description: "Block merges when PRs change filter validation logic without failing on invalid inputs" + enabled: true + severity: "high" + event_types: ["pull_request"] + parameters: + file_patterns: + - "packages/core/src/**/vector-query.ts" + - "packages/core/src/**/graph-rag.ts" + - "packages/core/src/**/filters/*.ts" + require_patterns: + - "throw\\s+new\\s+Error" + - "raise\\s+ValueError" + forbidden_patterns: + - "return\\s+.*filter\\s*$" + how_to_fix: "Ensure invalid filters raise descriptive errors instead of silently returning unfiltered results." + + - description: "Require regression tests when modifying tool schema validation or client tool execution" + enabled: true + severity: "medium" + event_types: ["pull_request"] + parameters: + source_patterns: + - "packages/core/src/**/tool*.ts" + - "packages/core/src/agent/**" + - "packages/client/**" + test_patterns: + - "packages/core/tests/**" + - "tests/**" + min_test_files: 1 + rationale: "Tool invocation changes have previously caused regressions in clientTools streaming." + + - description: "Ensure every agent exposes a user-facing description for UI profiles" + enabled: true + severity: "low" + event_types: ["pull_request"] + parameters: + file_patterns: + - "packages/core/src/agent/**" + required_text: + - "description" + message: "Add or update the agent description so downstream UIs can render capabilities." + + - description: "Block merges when URL or asset handling changes bypass provider capability checks" + enabled: true + severity: "high" + event_types: ["pull_request"] + parameters: + file_patterns: + - "packages/core/src/agent/message-list/**" + - "packages/core/src/llm/**" + require_patterns: + - "isUrlSupportedByModel" + forbidden_patterns: + - "downloadAssetsFromMessages\\(messages\\)" + how_to_fix: "Preserve remote URLs for providers that support them natively; only download assets for unsupported providers." diff --git a/docs/getting-started/configuration.md b/docs/getting-started/configuration.md index 9f51376..14bd4f1 100644 --- a/docs/getting-started/configuration.md +++ b/docs/getting-started/configuration.md @@ -111,6 +111,52 @@ parameters: excluded_branches: ["feature/*", "hotfix/*"] ``` +### Diff-Aware Validators + +Watchflow can now reason about pull-request diffs directly. The following parameter groups plug into diff-aware validators: + +#### `diff_pattern` + +Use this to require or forbid specific regex patterns inside matched files. + +```yaml +parameters: + file_patterns: + - "packages/core/src/**/vector-query.ts" + require_patterns: + - "throw\\s+new\\s+Error" + forbidden_patterns: + - "console\\.log" +``` + +#### `related_tests` + +Ensure core source changes include matching test updates. + +```yaml +parameters: + source_patterns: + - "packages/core/src/**" + test_patterns: + - "tests/**" + - "packages/core/tests/**" + min_test_files: 1 +``` + +#### `required_field_in_diff` + +Verify that additions to certain files include a text fragment (for example, enforcing `description` on new agents). + +```yaml +parameters: + file_patterns: + - "packages/core/src/agent/**" + required_text: + - "description" +``` + +These validators activate automatically when the parameters above are present, so you do not need to declare an `actions` block or manual mapping. + ## Severity Levels ### Severity Configuration @@ -219,6 +265,39 @@ rules: required_teams: ["senior-engineers"] ``` +## Diff-Aware Validators + +Watchflow supports advanced validators that inspect actual PR diffs to enforce code-level patterns: + +### diff_pattern +Enforce regex requirements or prohibitions within file patches. + +```yaml +parameters: + file_patterns: ["packages/core/src/**/vector-query.ts"] + require_patterns: ["throw\\s+new\\s+Error"] + forbid_patterns: ["silent.*skip"] +``` + +### related_tests +Require test file updates when core code changes. + +```yaml +parameters: + file_patterns: ["packages/core/src/**"] + require_test_updates: true + min_test_files: 1 +``` + +### required_field_in_diff +Ensure new additions include required fields (e.g., agent descriptions). + +```yaml +parameters: + file_patterns: ["packages/core/src/agent/**"] + required_text: "description" +``` + ## Best Practices ### Rule Design diff --git a/docs/reports/mastra-analysis.md b/docs/reports/mastra-analysis.md new file mode 100644 index 0000000..fce0901 --- /dev/null +++ b/docs/reports/mastra-analysis.md @@ -0,0 +1,133 @@ +# Mastra Repository Analysis + +Mastra (`mastra-ai/mastra`) is a TypeScript-first agent framework for building production-grade AI assistants. The project has roughly **280 contributors**, **134 open pull requests**, and active CI coverage via GitHub Actions. This document captures the agreed-upon analysis from November 2025 so we can align on rule proposals before shipping automation. + +## Repository Snapshot + +- **Focus**: AI agents with tooling, memory, workflows, and multi-step orchestration +- **Primary language**: TypeScript with pnpm-based monorepo +- **Governance signals**: Detailed `CONTRIBUTING.md`, CODEOWNERS, changeset automation, active doc set +- **Pain points**: Complex LLM/provider integrations, repeated validation gaps, and regression risk in shared tooling layers + +## Pull Request Sample (Nov 2025) + +| PR | Title | Outcome | Notes | +| --- | --- | --- | --- | +| [#10180](https://github.com/mastra-ai/mastra/pull/10180) | feat: add custom model gateway support with automatic type generation | ✅ merged | Large feature: gateway registry, TS type generation, doc updates | +| [#10269](https://github.com/mastra-ai/mastra/pull/10269) | AI SDK tripwire data chunks | ✅ merged | Fixes & changeset for SDK data chunking bug | +| [#10141](https://github.com/mastra-ai/mastra/pull/10141) | fix: throw on invalid filter instead of silently skipping filtering | ✅ merged | Addressed regression where invalid filters returned unfiltered data | +| [#10300](https://github.com/mastra-ai/mastra/pull/10300) | Add description to type | ✅ merged | Unblocked Agent profile UI by exposing description metadata | +| [#9880](https://github.com/mastra-ai/mastra/pull/9880) | Fix clientjs clientTools execution | ✅ merged | Fixed client-side tool streaming regressions | +| [#9941](https://github.com/mastra-ai/mastra/pull/9941) | fix(core): input tool validation with no schema | ✅ merged | Restored validation for schema-less tool inputs | + +## Pattern Summary + +- **Validation & safety gaps (≈40%)** – invalid filters or schema-less tools silently bypassed safeguards. +- **Tooling & integration regressions (≈33%)** – clientTools streaming, AI SDK data chunking, URL handling. +- **Experience polish gaps (≈17%)** – missing agent descriptions prevented UI consistency. +- **High merge velocity** – most fixes merged quickly; reinforces need for automated guardrails so regressions are caught before release. + +## Recommended Watchflow Rules + +Rules intentionally avoid the optional `actions:` block so they remain compatible with the current loader. Enforcement intent is described in each `description` and reflected in `severity`. + +```yaml +rules: + - description: "Block merges when PRs change filter validation logic without failing on invalid inputs" + enabled: true + severity: "high" + event_types: ["pull_request"] + parameters: + file_patterns: + - "packages/core/src/**/vector-query.ts" + - "packages/core/src/**/graph-rag.ts" + - "packages/core/src/**/filters/*.ts" + require_patterns: + - "throw\\s+new\\s+Error" + - "raise\\s+ValueError" + forbidden_patterns: + - "return\\s+.*filter\\s*$" + how_to_fix: "Ensure invalid filters raise descriptive errors instead of silently returning unfiltered results." + + - description: "Require regression tests when modifying tool schema validation or client tool execution" + enabled: true + severity: "medium" + event_types: ["pull_request"] + parameters: + source_patterns: + - "packages/core/src/**/tool*.ts" + - "packages/core/src/agent/**" + - "packages/client/**" + test_patterns: + - "packages/core/tests/**" + - "tests/**" + min_test_files: 1 + rationale: "Tool invocation changes have previously caused regressions in clientTools streaming." + + - description: "Ensure every agent exposes a user-facing description for UI profiles" + enabled: true + severity: "low" + event_types: ["pull_request"] + parameters: + file_patterns: + - "packages/core/src/agent/**" + required_text: + - "description" + message: "Add or update the agent description so downstream UIs can render capabilities." + + - description: "Block merges when URL or asset handling changes bypass provider capability checks" + enabled: true + severity: "high" + event_types: ["pull_request"] + parameters: + file_patterns: + - "packages/core/src/agent/message-list/**" + - "packages/core/src/llm/**" + require_patterns: + - "isUrlSupportedByModel" + forbidden_patterns: + - "downloadAssetsFromMessages\\(messages\\)" + how_to_fix: "Preserve remote URLs for providers that support them natively; only download assets for unsupported providers." +``` + +These concrete rules rely on the diff-aware validators recently added to Watchflow: + +- `diff_pattern` ensures critical patches keep throwing exceptions or performing capability checks. +- `related_tests` requires PRs touching core modules to include matching test updates. +- `required_field_in_diff` verifies additions to agent definitions include a `description` so downstream UIs stay in sync. + +Because the PR processor now passes normalized diffs into the engine, these validators operate deterministically without LLM fallbacks. + +## PR Template Snippet + +```markdown +## Repository Analysis Complete + +We've analyzed your repository and identified key quality patterns based on recent PR history. + +### Key Findings +- 40% of recent fixes patched validation or data-safety gaps (filters, schema-less tools). +- 33% addressed tool/LLM integration regressions (clientTools, AI SDK, URL handling). +- Tests/documentation often lag behind critical fixes, creating follow-up churn. + +### Recommended Rules +- Block filter-validation changes that stop throwing on invalid inputs. +- Require regression tests when modifying tool schemas or clientTools execution. +- Enforce agent descriptions so UI consumers can present profiles. +- Block URL/asset handling changes that skip provider capability checks. + +### Installation +1. Install the Watchflow GitHub App and grant access to `mastra-ai/mastra`. +2. Add `.watchflow/rules.yaml` with the rules above (see snippet). +3. Watchflow will start reporting violations through status checks immediately. + +Questions? Reach out to the Watchflow team. +``` + +## Validation Plan + +1. Keep the rule definitions in `docs/samples/mastra-watchflow-rules.yaml`. +2. Run `pytest tests/unit/test_mastra_rules_sample.py` to ensure every rule loads via `Rule.model_validate`. +3. (Optional) Use the repository analysis agent once PR-diff ingestion ships to simulate Mastra commits before opening an automated PR with these rules. + +This keeps the deliverable lightweight, fully tested, and ready for the PR template automation flow discussed with Dimitris. diff --git a/mkdocs.yml b/mkdocs.yml index 8298626..2497c6c 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -67,6 +67,8 @@ nav: - Comparative Analysis: benchmarks.md - Architecture: - Overview: concepts/overview.md + - Case Studies: + - Mastra Repository Analysis: reports/mastra-analysis.md # Plugins plugins: @@ -132,6 +134,3 @@ extra: social: - icon: fontawesome/brands/github link: https://github.com/warestack/watchflow - analytics: - provider: google - property: ${GOOGLE_ANALYTICS_KEY} diff --git a/src/agents/engine_agent/prompts.py b/src/agents/engine_agent/prompts.py index 9c9f5f9..15b5c03 100644 --- a/src/agents/engine_agent/prompts.py +++ b/src/agents/engine_agent/prompts.py @@ -177,7 +177,7 @@ def _extract_event_context(event_data: dict, event_type: str) -> str: context_parts = [] if event_type == "pull_request": - pr = event_data.get("pull_request", {}) + pr = event_data.get("pull_request_details") or event_data.get("pull_request") or {} context_parts.extend( [ f"Title: {pr.get('title', 'N/A')}", @@ -188,6 +188,17 @@ def _extract_event_context(event_data: dict, event_type: str) -> str: ] ) + files = event_data.get("files", []) + if files: + top_files = [file.get("filename") for file in files[:5] if file.get("filename")] + context_parts.append( + f"Changed Files ({len(files)} total): {top_files if top_files else '[filenames unavailable]'}" + ) + + diff_summary = event_data.get("diff_summary") + if diff_summary: + context_parts.append(f"Diff Summary:\n{diff_summary}") + elif event_type == "push": context_parts.extend( [ diff --git a/src/agents/factory.py b/src/agents/factory.py index e7fa353..df270a3 100644 --- a/src/agents/factory.py +++ b/src/agents/factory.py @@ -12,6 +12,7 @@ from src.agents.base import BaseAgent from src.agents.engine_agent import RuleEngineAgent from src.agents.feasibility_agent import RuleFeasibilityAgent +from src.agents.repository_analysis_agent import RepositoryAnalysisAgent logger = logging.getLogger(__name__) @@ -34,6 +35,7 @@ def get_agent(agent_type: str, **kwargs: Any) -> BaseAgent: >>> engine_agent = get_agent("engine") >>> feasibility_agent = get_agent("feasibility") >>> acknowledgment_agent = get_agent("acknowledgment") + >>> analysis_agent = get_agent("repository_analysis") """ agent_type = agent_type.lower() @@ -43,6 +45,8 @@ def get_agent(agent_type: str, **kwargs: Any) -> BaseAgent: return RuleFeasibilityAgent(**kwargs) elif agent_type == "acknowledgment": return AcknowledgmentAgent(**kwargs) + elif agent_type == "repository_analysis": + return RepositoryAnalysisAgent(**kwargs) else: - supported = ", ".join(["engine", "feasibility", "acknowledgment"]) + supported = ", ".join(["engine", "feasibility", "acknowledgment", "repository_analysis"]) raise ValueError(f"Unsupported agent type: {agent_type}. Supported: {supported}") diff --git a/src/agents/repository_analysis_agent/README.md b/src/agents/repository_analysis_agent/README.md new file mode 100644 index 0000000..b834203 --- /dev/null +++ b/src/agents/repository_analysis_agent/README.md @@ -0,0 +1,86 @@ +# Repository Analysis Agent + +the Repository Analysis Agent analyzes GitHub repositories to generate personalized Watchflow rule recommendations based on repository structure, contributing guidelines, and development patterns. + +## Overview + +this agent performs comprehensive analysis of repositories and provides actionable governance rule recommendations with confidence scores and reasoning. + +## Features + +- **Repository Structure Analysis**: Examines workflows, branch protection, contributors, and repository metadata +- **Contributing Guidelines Parsing**: Uses LLM analysis to extract requirements from CONTRIBUTING.md files +- **Pattern-Based Recommendations**: Generates rules based on detected repository characteristics +- **Confidence Scoring**: Each recommendation includes a confidence score (0.0-1.0) and reasoning +- **Valid YAML Generation**: All recommendations are valid Watchflow rule YAML + +## Usage + +### Direct Agent Usage + +```python +from src.agents import get_agent + + +agent = get_agent("repository_analysis") + + +result = await agent.execute( + repository_full_name="owner/repo-name", + installation_id=12345 +) + + +response = result.data["analysis_response"] +for recommendation in response.recommendations: + print(f"Confidence: {recommendation.confidence}") + print(f"Category: {recommendation.category}") + print(f"Reasoning: {recommendation.reasoning}") + print(f"YAML:\n{recommendation.yaml_content}") +``` + + + +## Recommendation Categories + +The agent generates recommendations in the following categories: + +- **Quality**: Code quality rules (linting, testing, CI/CD) +- **Security**: Security-focused rules (dependency scanning, secrets detection) +- **Process**: Development process rules (reviews, approvals, branch protection) +- **Documentation**: Documentation-related rules (README updates, CHANGELOG) + +## Analysis Workflow + +The agent follows a multi-step LangGraph workflow: + +1. **Repository Structure Analysis**: Gathers basic repository metadata +2. **Contributing Guidelines Analysis**: Parses CONTRIBUTING.md for requirements +3. **Rule Generation**: Creates recommendations based on detected patterns +4. **Validation**: Ensures all recommendations contain valid YAML +5. **Summarization**: Provides analysis summary and statistics + +## Configuration + +The agent can be configured with the following parameters: + +- `max_retries`: Maximum retry attempts for LLM calls (default: 3) +- `timeout`: Maximum execution time in seconds (default: 120.0) + +```python +agent = get_agent("repository_analysis", max_retries=5, timeout=300.0) +``` + +## Caching and Rate Limiting + +The API endpoint includes: +- **Caching**: Successful analyses are cached for 1 hour +- **Rate Limiting**: Basic rate limiting to prevent abuse +- **Error Handling**: Comprehensive error handling with structured logging + + + + +## Integration with Watchflow.dev + +This agent provides the backend for watchflow.dev's onboarding flow, automatically suggesting appropriate governance rules based on repository analysis. diff --git a/src/agents/repository_analysis_agent/__init__.py b/src/agents/repository_analysis_agent/__init__.py new file mode 100644 index 0000000..e789f07 --- /dev/null +++ b/src/agents/repository_analysis_agent/__init__.py @@ -0,0 +1,10 @@ +""" +Repository Analysis Agent for generating Watchflow rule recommendations. + +This agent analyzes repository structure, contributing guidelines, and patterns +to automatically propose appropriate Watchflow rules with confidence scores. +""" + +from src.agents.repository_analysis_agent.agent import RepositoryAnalysisAgent + +__all__ = ["RepositoryAnalysisAgent"] diff --git a/src/agents/repository_analysis_agent/agent.py b/src/agents/repository_analysis_agent/agent.py new file mode 100644 index 0000000..f2cf48d --- /dev/null +++ b/src/agents/repository_analysis_agent/agent.py @@ -0,0 +1,180 @@ +import logging +import time +from datetime import datetime + +from langgraph.graph import END, START, StateGraph + +from src.agents.base import AgentResult, BaseAgent +from src.agents.repository_analysis_agent.models import ( + RepositoryAnalysisRequest, + RepositoryAnalysisResponse, + RepositoryAnalysisState, +) +from src.agents.repository_analysis_agent.nodes import ( + analyze_contributing_guidelines, + analyze_pr_history, + analyze_repository_structure, + generate_rule_recommendations, + summarize_analysis, + validate_recommendations, +) + +logger = logging.getLogger(__name__) + + +class RepositoryAnalysisAgent(BaseAgent): + """ + Agent that analyzes GitHub repositories to generate Watchflow rule recommendations. + + This agent performs multi-step analysis: + 1. Analyzes repository structure and features + 2. Parses contributing guidelines for patterns + 3. Reviews commit/PR patterns + 4. Generates rule recommendations with confidence scores + 5. Validates recommendations are valid YAML + + Returns structured recommendations that can be directly used as Watchflow rules. + """ + + def __init__(self, max_retries: int = 3, timeout: float = 120.0): + super().__init__(max_retries=max_retries, agent_name="repository_analysis_agent") + self.timeout = timeout + + logger.info("Repository Analysis Agent initialized") + logger.info(f"Max retries: {max_retries}, Timeout: {timeout}s") + + def _build_graph(self) -> StateGraph: + """Build the LangGraph workflow for repository analysis.""" + workflow = StateGraph(RepositoryAnalysisState) + + # Add nodes + workflow.add_node("analyze_repository_structure", analyze_repository_structure) + workflow.add_node("analyze_pr_history", analyze_pr_history) + workflow.add_node("analyze_contributing_guidelines", analyze_contributing_guidelines) + workflow.add_node("generate_rule_recommendations", generate_rule_recommendations) + workflow.add_node("validate_recommendations", validate_recommendations) + workflow.add_node("summarize_analysis", summarize_analysis) + + # Define workflow edges + workflow.add_edge(START, "analyze_repository_structure") + workflow.add_edge("analyze_repository_structure", "analyze_pr_history") + workflow.add_edge("analyze_pr_history", "analyze_contributing_guidelines") + workflow.add_edge("analyze_contributing_guidelines", "generate_rule_recommendations") + workflow.add_edge("generate_rule_recommendations", "validate_recommendations") + workflow.add_edge("validate_recommendations", "summarize_analysis") + workflow.add_edge("summarize_analysis", END) + + return workflow.compile() + + async def execute(self, repository_full_name: str, installation_id: int | None = None, **kwargs) -> AgentResult: + """ + Analyze a repository and generate rule recommendations. + + Args: + repository_full_name: Full repository name (owner/repo) + installation_id: Optional GitHub App installation ID for private repos + **kwargs: Additional parameters + + Returns: + AgentResult containing analysis results and recommendations + """ + start_time = time.time() + + try: + logger.info(f"Starting repository analysis for {repository_full_name}") + + # Validate input + if not repository_full_name or "/" not in repository_full_name: + return AgentResult( + success=False, + message="Invalid repository name format. Expected 'owner/repo'", + data={}, + metadata={"execution_time_ms": 0}, + ) + + initial_state = RepositoryAnalysisState( + repository_full_name=repository_full_name, + installation_id=installation_id, + analysis_steps=[], + errors=[], + ) + + logger.info("Initial state prepared, starting analysis workflow") + + result = await self._execute_with_timeout(self.graph.ainvoke(initial_state), timeout=self.timeout) + + execution_time = time.time() - start_time + logger.info(f"Analysis completed in {execution_time:.2f}s") + + if isinstance(result, dict): + state = RepositoryAnalysisState(**result) + else: + state = result + + response = RepositoryAnalysisResponse( + repository_full_name=repository_full_name, + recommendations=state.recommendations, + analysis_summary=state.analysis_summary, + analyzed_at=datetime.now().isoformat(), + total_recommendations=len(state.recommendations), + ) + + # Check for errors + has_errors = len(state.errors) > 0 + success_message = f"Analysis completed successfully with {len(state.recommendations)} recommendations" + if has_errors: + success_message += f" ({len(state.errors)} errors encountered)" + + logger.info(f"Analysis result: {len(state.recommendations)} recommendations, {len(state.errors)} errors") + + return AgentResult( + success=not has_errors, + message=success_message, + data={"analysis_response": response}, + metadata={ + "execution_time_ms": execution_time * 1000, + "recommendations_count": len(state.recommendations), + "errors_count": len(state.errors), + "analysis_steps": state.analysis_steps, + }, + ) + + except Exception as e: + execution_time = time.time() - start_time + logger.error(f"Error in repository analysis: {e}") + + return AgentResult( + success=False, + message=f"Repository analysis failed: {str(e)}", + data={}, + metadata={ + "execution_time_ms": execution_time * 1000, + "error_type": type(e).__name__, + }, + ) + + async def analyze_repository(self, request: RepositoryAnalysisRequest) -> RepositoryAnalysisResponse: + """ + Convenience method for analyzing a repository using the request model. + + Args: + request: Repository analysis request + + Returns: + Repository analysis response + """ + result = await self.execute( + repository_full_name=request.repository_full_name, + installation_id=request.installation_id, + ) + + if result.success and "analysis_response" in result.data: + return result.data["analysis_response"] + else: + return RepositoryAnalysisResponse( + repository_full_name=request.repository_full_name, + recommendations=[], + analysis_summary={"error": result.message}, + analyzed_at=datetime.now().isoformat(), + total_recommendations=0, + ) diff --git a/src/agents/repository_analysis_agent/models.py b/src/agents/repository_analysis_agent/models.py new file mode 100644 index 0000000..e7d6b27 --- /dev/null +++ b/src/agents/repository_analysis_agent/models.py @@ -0,0 +1,95 @@ +from enum import Enum +from typing import Any + +from pydantic import BaseModel, Field + + +class AnalysisSource(str, Enum): + """Sources of analysis data for rule recommendations.""" + + CONTRIBUTING_GUIDELINES = "contributing_guidelines" + REPOSITORY_STRUCTURE = "repository_structure" + WORKFLOWS = "workflows" + BRANCH_PROTECTION = "branch_protection" + COMMIT_PATTERNS = "commit_patterns" + PR_PATTERNS = "pr_patterns" + + +class RuleRecommendation(BaseModel): + """A recommended Watchflow rule with confidence and reasoning.""" + + yaml_content: str = Field(description="Valid Watchflow rule YAML content") + confidence: float = Field(description="Confidence score (0.0-1.0) in the recommendation", ge=0.0, le=1.0) + reasoning: str = Field(description="Explanation of why this rule is recommended") + source_patterns: list[str] = Field( + description="Repository patterns that led to this recommendation", default_factory=list + ) + category: str = Field(description="Category of the rule (e.g., 'quality', 'security', 'process')") + estimated_impact: str = Field(description="Expected impact (e.g., 'high', 'medium', 'low')") + + +class RepositoryAnalysisRequest(BaseModel): + """Request model for repository analysis.""" + + repository_full_name: str = Field(description="Full repository name (owner/repo)") + installation_id: int | None = Field( + description="GitHub App installation ID for accessing private repos", default=None + ) + + +class RepositoryFeatures(BaseModel): + """Features and characteristics discovered in the repository.""" + + has_contributing: bool = Field(description="Has CONTRIBUTING.md file", default=False) + has_codeowners: bool = Field(description="Has CODEOWNERS file", default=False) + has_workflows: bool = Field(description="Has GitHub Actions workflows", default=False) + has_branch_protection: bool = Field(description="Has branch protection rules", default=False) + workflow_count: int = Field(description="Number of workflow files", default=0) + language: str | None = Field(description="Primary programming language", default=None) + contributor_count: int = Field(description="Number of contributors", default=0) + pr_count: int = Field(description="Number of pull requests", default=0) + issue_count: int = Field(description="Number of issues", default=0) + + +class ContributingGuidelinesAnalysis(BaseModel): + """Analysis of contributing guidelines content.""" + + content: str | None = Field(description="Full CONTRIBUTING.md content", default=None) + has_pr_template: bool = Field(description="Requires PR templates", default=False) + has_issue_template: bool = Field(description="Requires issue templates", default=False) + requires_tests: bool = Field(description="Requires tests for contributions", default=False) + requires_docs: bool = Field(description="Requires documentation updates", default=False) + code_style_requirements: list[str] = Field(description="Code style requirements mentioned", default_factory=list) + review_requirements: list[str] = Field(description="Code review requirements mentioned", default_factory=list) + + +class RepositoryAnalysisState(BaseModel): + """State for the repository analysis workflow.""" + + repository_full_name: str + installation_id: int | None + pr_samples: list[dict[str, Any]] = Field(default_factory=list) + + # Analysis data + repository_features: RepositoryFeatures = Field(default_factory=RepositoryFeatures) + contributing_analysis: ContributingGuidelinesAnalysis = Field(default_factory=ContributingGuidelinesAnalysis) + + # Processing state + analysis_steps: list[str] = Field(default_factory=list) + errors: list[str] = Field(default_factory=list) + + # Results + recommendations: list[RuleRecommendation] = Field(default_factory=list) + analysis_summary: dict[str, Any] = Field(default_factory=dict) + + +class RepositoryAnalysisResponse(BaseModel): + """Response model containing rule recommendations.""" + + repository_full_name: str = Field(description="Repository that was analyzed") + recommendations: list[RuleRecommendation] = Field( + description="List of recommended Watchflow rules", default_factory=list + ) + analysis_summary: dict[str, Any] = Field(description="Summary of analysis findings", default_factory=dict) + analyzed_at: str = Field(description="Timestamp of analysis") + total_recommendations: int = Field(description="Total number of recommendations made") diff --git a/src/agents/repository_analysis_agent/nodes.py b/src/agents/repository_analysis_agent/nodes.py new file mode 100644 index 0000000..e83a50d --- /dev/null +++ b/src/agents/repository_analysis_agent/nodes.py @@ -0,0 +1,442 @@ +import logging +from typing import Any + +from src.agents.repository_analysis_agent.models import ( + ContributingGuidelinesAnalysis, + RepositoryAnalysisState, + RepositoryFeatures, + RuleRecommendation, +) +from src.agents.repository_analysis_agent.prompts import ( + CONTRIBUTING_GUIDELINES_ANALYSIS_PROMPT, +) +from src.integrations.github.api import github_client + +logger = logging.getLogger(__name__) + + +async def analyze_repository_structure(state: RepositoryAnalysisState) -> dict[str, Any]: + """ + Analyze basic repository structure and features. + + Gathers information about workflows, branch protection, contributors, etc. + """ + try: + logger.info(f"Analyzing repository structure for {state.repository_full_name}") + + features = RepositoryFeatures() + contributing_content = await github_client.get_file_content( + state.repository_full_name, "CONTRIBUTING.md", state.installation_id + ) + features.has_contributing = contributing_content is not None + + codeowners_content = await github_client.get_file_content( + state.repository_full_name, ".github/CODEOWNERS", state.installation_id + ) + features.has_codeowners = codeowners_content is not None + + workflow_content = await github_client.get_file_content( + state.repository_full_name, ".github/workflows/main.yml", state.installation_id + ) + if workflow_content: + features.has_workflows = True + features.workflow_count = 1 + + contributors = await github_client.get_repository_contributors( + state.repository_full_name, state.installation_id + ) + features.contributor_count = len(contributors) if contributors else 0 + + # TODO: Add more repository analysis (PR count, issues, language detection, etc.) + + logger.info(f"Repository analysis complete: {features.model_dump()}") + + state.repository_features = features + state.analysis_steps.append("repository_structure_analyzed") + + return {"repository_features": features, "analysis_steps": state.analysis_steps} + + except Exception as e: + logger.error(f"Error analyzing repository structure: {e}") + state.errors.append(f"Repository structure analysis failed: {str(e)}") + return {"errors": state.errors} + + +async def analyze_pr_history(state: RepositoryAnalysisState) -> dict[str, Any]: + """Pull a small PR sample to inform rule recommendations.""" + try: + logger.info(f"Fetching recent PRs for {state.repository_full_name}") + prs = await github_client.list_pull_requests( + state.repository_full_name, state.installation_id or 0, state="closed", per_page=20 + ) + + pr_samples: list[dict[str, Any]] = [] + for pr in prs: + pr_samples.append( + { + "number": pr.get("number"), + "title": pr.get("title"), + "merged": pr.get("merged_at") is not None, + "changed_files": pr.get("changed_files"), + "additions": pr.get("additions"), + "deletions": pr.get("deletions"), + "user": pr.get("user", {}).get("login"), + } + ) + + state.pr_samples = pr_samples + state.analysis_steps.append("pr_history_sampled") + logger.info(f"Collected {len(pr_samples)} PR samples") + return {"pr_samples": pr_samples, "analysis_steps": state.analysis_steps} + except Exception as e: + logger.error(f"Error analyzing PR history: {e}") + state.errors.append(f"PR history analysis failed: {str(e)}") + return {"errors": state.errors} + + +async def analyze_contributing_guidelines(state: RepositoryAnalysisState) -> dict[str, Any]: + """ + Analyze CONTRIBUTING.md file for patterns and requirements. + """ + try: + logger.info(f" Analyzing contributing guidelines for {state.repository_full_name}") + + # Get contributing guidelines content + content = await github_client.get_file_content( + state.repository_full_name, "CONTRIBUTING.md", state.installation_id + ) + + if not content: + logger.info("No CONTRIBUTING.md file found") + analysis = ContributingGuidelinesAnalysis() + else: + llm = github_client.llm if hasattr(github_client, "llm") else None + if llm: + try: + prompt = CONTRIBUTING_GUIDELINES_ANALYSIS_PROMPT.format(content=content) + await llm.ainvoke(prompt) + + # TODO: Parse JSON response and create ContributingGuidelinesAnalysis + + analysis = ContributingGuidelinesAnalysis(content=content) + except Exception as e: + logger.error(f"LLM analysis failed: {e}") + analysis = ContributingGuidelinesAnalysis(content=content) + else: + analysis = ContributingGuidelinesAnalysis(content=content) + + state.contributing_analysis = analysis + state.analysis_steps.append("contributing_guidelines_analyzed") + + logger.info(" Contributing guidelines analysis complete") + + return {"contributing_analysis": analysis, "analysis_steps": state.analysis_steps} + + except Exception as e: + logger.error(f"Error analyzing contributing guidelines: {e}") + state.errors.append(f"Contributing guidelines analysis failed: {str(e)}") + return {"errors": state.errors} + + +async def generate_rule_recommendations(state: RepositoryAnalysisState) -> dict[str, Any]: + """ + Generate Watchflow rule recommendations based on repository analysis. + """ + try: + logger.info(f" Generating rule recommendations for {state.repository_full_name}") + + recommendations = [] + + features = state.repository_features + contributing = state.contributing_analysis + + # Diff-aware: enforce filter handling in core RAG/query code + recommendations.append( + RuleRecommendation( + yaml_content="""description: "Block merges when PRs change filter validation logic without failing on invalid inputs" +enabled: true +severity: "high" +event_types: ["pull_request"] +parameters: + file_patterns: + - "packages/core/src/**/vector-query.ts" + - "packages/core/src/**/graph-rag.ts" + - "packages/core/src/**/filters/*.ts" + require_patterns: + - "throw\\\\s+new\\\\s+Error" + - "raise\\\\s+ValueError" + forbidden_patterns: + - "return\\\\s+.*filter\\\\s*$" + how_to_fix: "Ensure invalid filters raise descriptive errors instead of silently returning unfiltered results." +""", + confidence=0.85, + reasoning="Filter handling regressions were flagged in historical fixes; enforce throws on invalid input.", + source_patterns=["pr_history"], + category="quality", + estimated_impact="high", + ) + ) + + # Diff-aware: enforce test updates when core code changes + recommendations.append( + RuleRecommendation( + yaml_content="""description: "Require regression tests when modifying tool schema validation or client tool execution" +enabled: true +severity: "medium" +event_types: ["pull_request"] +parameters: + source_patterns: + - "packages/core/src/**/tool*.ts" + - "packages/core/src/agent/**" + - "packages/client/**" + test_patterns: + - "packages/core/tests/**" + - "tests/**" + min_test_files: 1 + rationale: "Tool invocation changes have previously caused regressions in clientTools streaming." +""", + confidence=0.8, + reasoning="Core tool changes often broke client tools; require at least one related test update.", + source_patterns=["pr_history"], + category="quality", + estimated_impact="medium", + ) + ) + + # Diff-aware: ensure agent descriptions exist + recommendations.append( + RuleRecommendation( + yaml_content="""description: "Ensure every agent exposes a user-facing description for UI profiles" +enabled: true +severity: "low" +event_types: ["pull_request"] +parameters: + file_patterns: + - "packages/core/src/agent/**" + required_text: + - "description" + message: "Add or update the agent description so downstream UIs can render capabilities." +""", + confidence=0.75, + reasoning="Agent profile UIs require descriptions; ensure new/updated agents include them.", + source_patterns=["pr_history"], + category="process", + estimated_impact="low", + ) + ) + + # Diff-aware: preserve URL handling for supported providers + recommendations.append( + RuleRecommendation( + yaml_content="""description: "Block merges when URL or asset handling changes bypass provider capability checks" +enabled: true +severity: "high" +event_types: ["pull_request"] +parameters: + file_patterns: + - "packages/core/src/agent/message-list/**" + - "packages/core/src/llm/**" + require_patterns: + - "isUrlSupportedByModel" + forbidden_patterns: + - "downloadAssetsFromMessages\\(messages\\)" + how_to_fix: "Preserve remote URLs for providers that support them natively; only download assets for unsupported providers." +""", + confidence=0.8, + reasoning="Past URL handling bugs; ensure capability checks remain intact.", + source_patterns=["pr_history"], + category="quality", + estimated_impact="high", + ) + ) + + # Legacy structural signals retained for completeness + if features.has_workflows: + recommendations.append( + RuleRecommendation( + yaml_content="""description: "Require CI checks to pass" +enabled: true +severity: "high" +event_types: + - pull_request +conditions: + - type: "ci_checks_passed" + parameters: + required_checks: [] +actions: + - type: "block_merge" + parameters: + message: "All CI checks must pass before merging" +""", + confidence=0.9, + reasoning="Repository has CI workflows configured, so requiring checks to pass is a standard practice", + source_patterns=["has_workflows"], + category="quality", + estimated_impact="high", + ) + ) + + if features.has_codeowners: + recommendations.append( + RuleRecommendation( + yaml_content="""description: "Require CODEOWNERS approval for changes" +enabled: true +severity: "medium" +event_types: + - pull_request +conditions: + - type: "codeowners_approved" + parameters: {} +actions: + - type: "require_approval" + parameters: + message: "CODEOWNERS must approve changes to owned files" +""", + confidence=0.8, + reasoning="CODEOWNERS file exists, indicating ownership requirements for code changes", + source_patterns=["has_codeowners"], + category="process", + estimated_impact="medium", + ) + ) + + if contributing.requires_tests: + recommendations.append( + RuleRecommendation( + yaml_content="""description: "Require test coverage for code changes" +enabled: true +severity: "medium" +event_types: + - pull_request +conditions: + - type: "test_coverage_threshold" + parameters: + minimum_coverage: 80 +actions: + - type: "block_merge" + parameters: + message: "Test coverage must be at least 80%" +""", + confidence=0.7, + reasoning="Contributing guidelines mention testing requirements", + source_patterns=["requires_tests"], + category="quality", + estimated_impact="medium", + ) + ) + + if features.contributor_count > 10: + recommendations.append( + RuleRecommendation( + yaml_content="""description: "Require at least one approval for pull requests" +enabled: true +severity: "medium" +event_types: + - pull_request +conditions: + - type: "minimum_approvals" + parameters: + count: 1 +actions: + - type: "block_merge" + parameters: + message: "Pull requests require at least one approval" +""", + confidence=0.6, + reasoning="Repository has multiple contributors, indicating collaborative development", + source_patterns=["contributor_count"], + category="process", + estimated_impact="medium", + ) + ) + + state.recommendations = recommendations + state.analysis_steps.append("recommendations_generated") + + logger.info(f"Generated {len(recommendations)} rule recommendations") + + return {"recommendations": recommendations, "analysis_steps": state.analysis_steps} + + except Exception as e: + logger.error(f"Error generating recommendations: {e}") + state.errors.append(f"Recommendation generation failed: {str(e)}") + return {"errors": state.errors} + + +async def validate_recommendations(state: RepositoryAnalysisState) -> dict[str, Any]: + """ + Validate that generated recommendations contain valid YAML. + """ + try: + logger.info("Validating rule recommendations") + + import yaml + + valid_recommendations = [] + + for rec in state.recommendations: + try: + # Parse YAML to validate syntax + parsed = yaml.safe_load(rec.yaml_content) + if parsed and isinstance(parsed, dict): + valid_recommendations.append(rec) + else: + logger.warning(f"Invalid rule structure: {rec.yaml_content[:100]}...") + except yaml.YAMLError as e: + logger.error(f"Invalid YAML in recommendation: {e}") + continue + + state.recommendations = valid_recommendations + state.analysis_steps.append("recommendations_validated") + + logger.info(f"Validated {len(valid_recommendations)} recommendations") + + return {"recommendations": valid_recommendations, "analysis_steps": state.analysis_steps} + + except Exception as e: + logger.error(f"Error validating recommendations: {e}") + state.errors.append(f"Recommendation validation failed: {str(e)}") + return {"errors": state.errors} + + +async def summarize_analysis(state: RepositoryAnalysisState) -> dict[str, Any]: + """ + Create a summary of the analysis findings. + """ + try: + logger.info("Creating analysis summary") + + summary = { + "repository": state.repository_full_name, + "features_analyzed": { + "has_contributing": state.repository_features.has_contributing, + "has_codeowners": state.repository_features.has_codeowners, + "has_workflows": state.repository_features.has_workflows, + "contributor_count": state.repository_features.contributor_count, + }, + "recommendations_count": len(state.recommendations), + "recommendations_by_category": {}, + "high_confidence_count": 0, + "analysis_steps_completed": len(state.analysis_steps), + "errors_encountered": len(state.errors), + } + + # Count recommendations by category + for rec in state.recommendations: + summary["recommendations_by_category"][rec.category] = ( + summary["recommendations_by_category"].get(rec.category, 0) + 1 + ) + if rec.confidence >= 0.8: + summary["high_confidence_count"] += 1 + + state.analysis_summary = summary + state.analysis_steps.append("analysis_summarized") + + logger.info("Analysis summary created") + + return {"analysis_summary": summary, "analysis_steps": state.analysis_steps} + + except Exception as e: + logger.error(f"Error creating analysis summary: {e}") + state.errors.append(f"Analysis summary failed: {str(e)}") + return {"errors": state.errors} diff --git a/src/agents/repository_analysis_agent/prompts.py b/src/agents/repository_analysis_agent/prompts.py new file mode 100644 index 0000000..94bfe4a --- /dev/null +++ b/src/agents/repository_analysis_agent/prompts.py @@ -0,0 +1,97 @@ +from langchain_core.prompts import ChatPromptTemplate + +CONTRIBUTING_GUIDELINES_ANALYSIS_PROMPT = ChatPromptTemplate.from_template(""" +You are a senior software engineer analyzing contributing guidelines to recommend appropriate repository governance rules. + +Analyze the following CONTRIBUTING.md content and extract patterns, requirements, and best practices that would benefit from automated enforcement via Watchflow rules. + +CONTRIBUTING.md Content: +{content} + +Your task is to extract: +1. Pull request requirements (templates, reviews, tests, etc.) +2. Code quality standards (linting, formatting, etc.) +3. Documentation requirements +4. Commit message conventions +5. Branch naming conventions +6. Testing requirements +7. Security practices + +Provide your analysis in the following JSON format: +{{ + "has_pr_template": boolean, + "has_issue_template": boolean, + "requires_tests": boolean, + "requires_docs": boolean, + "code_style_requirements": ["list", "of", "requirements"], + "review_requirements": ["list", "of", "requirements"] +}} + +Be thorough but only extract information that is explicitly mentioned or strongly implied in the guidelines. +""") + +REPOSITORY_ANALYSIS_PROMPT = ChatPromptTemplate.from_template(""" +You are analyzing a GitHub repository to recommend Watchflow rules based on its structure, workflows, and contributing patterns. + +Repository Information: +- Name: {repository_full_name} +- Primary Language: {language} +- Contributors: {contributor_count} +- Pull Requests: {pr_count} +- Issues: {issue_count} +- Has Workflows: {has_workflows} +- Has Branch Protection: {has_branch_protection} +- Has CODEOWNERS: {has_codeowners} + +Contributing Guidelines Analysis: +{contributing_analysis} + +Based on this repository profile, recommend appropriate Watchflow rules that would improve governance, quality, and security. + +Consider: +1. Code quality rules (linting, testing, formatting) +2. Security rules (dependency scanning, secret detection) +3. Process rules (PR reviews, branch protection, CI/CD) +4. Documentation rules (README updates, CHANGELOG) + +For each recommendation, provide: +- A valid Watchflow rule YAML +- Confidence score (0.0-1.0) +- Reasoning for the recommendation +- Source patterns that led to it +- Category and impact level + +Focus on rules that are most relevant to this repository's characteristics and would provide the most value. +""") + +RULE_GENERATION_PROMPT = ChatPromptTemplate.from_template(""" +Generate a valid Watchflow rule YAML based on the following specification: + +Category: {category} +Description: {description} +Parameters: {parameters} +Event Types: {event_types} +Severity: {severity} + +Generate a complete, valid Watchflow rule in YAML format that implements this specification. +Ensure the rule follows Watchflow YAML schema and is properly formatted. + +Watchflow Rule YAML Format: +```yaml +description: "Rule description" +enabled: true +severity: "medium" +event_types: + - pull_request +conditions: + - type: "condition_type" + parameters: + key: "value" +actions: + - type: "action_type" + parameters: + key: "value" +``` + +Make sure the rule is functional and follows best practices. +""") diff --git a/src/agents/repository_analysis_agent/test_agent.py b/src/agents/repository_analysis_agent/test_agent.py new file mode 100644 index 0000000..8b0a104 --- /dev/null +++ b/src/agents/repository_analysis_agent/test_agent.py @@ -0,0 +1,159 @@ +from unittest.mock import AsyncMock, patch + +import pytest + +from src.agents.repository_analysis_agent.agent import RepositoryAnalysisAgent +from src.agents.repository_analysis_agent.models import ( + RepositoryAnalysisRequest, + RepositoryAnalysisResponse, + RepositoryFeatures, + RuleRecommendation, +) + + +class TestRepositoryAnalysisAgent: + """Test cases for RepositoryAnalysisAgent.""" + + @pytest.fixture + def agent(self): + """Create a test instance of RepositoryAnalysisAgent.""" + return RepositoryAnalysisAgent(max_retries=1, timeout=30.0) + + @pytest.mark.asyncio + async def test_execute_invalid_repository_name(self, agent): + """Test that invalid repository names are rejected.""" + result = await agent.execute("invalid-repo-name") + + assert not result.success + assert "Invalid repository name format" in result.message + + @pytest.mark.asyncio + async def test_execute_with_mock_github_client(self, agent): + """Test repository analysis with mocked GitHub client.""" + + with patch("src.agents.repository_analysis_agent.nodes.github_client") as mock_client: + mock_client.get_file_content = AsyncMock( + side_effect=[ + None, # CONTRIBUTING.md not found + None, # .github/CODEOWNERS not found + None, # workflow file not found + ] + ) + mock_client.get_repository_contributors = AsyncMock( + return_value=[ + {"login": "user1", "contributions": 10}, + {"login": "user2", "contributions": 5}, + ] + ) + + result = await agent.execute("test-owner/test-repo") + + assert result.success + assert "analysis_response" in result.data + + response = result.data["analysis_response"] + assert isinstance(response, RepositoryAnalysisResponse) + assert response.repository_full_name == "test-owner/test-repo" + assert isinstance(response.recommendations, list) + assert isinstance(response.analysis_summary, dict) + + @pytest.mark.asyncio + async def test_analyze_repository_with_contributing_file(self, agent): + """Test analysis when CONTRIBUTING.md exists.""" + with patch("src.agents.repository_analysis_agent.nodes.github_client") as mock_client: + mock_client.get_file_content = AsyncMock( + side_effect=[ + "# Contributing Guidelines\n\n## Testing\nAll PRs must include tests.", # CONTRIBUTING.md + None, # CODEOWNERS + None, # workflow + ] + ) + mock_client.get_repository_contributors = AsyncMock(return_value=[]) + + result = await agent.execute("test-owner/test-repo") + + assert result.success + response = result.data["analysis_response"] + + assert len(response.recommendations) > 0 + + assert response.analysis_summary["features_analyzed"]["has_contributing"] is True + + def test_workflow_structure(self, agent): + """Test that the LangGraph workflow is properly structured.""" + graph = agent.graph + + assert hasattr(graph, "nodes") + + @pytest.mark.asyncio + async def test_error_handling(self, agent): + """Test error handling in repository analysis.""" + with patch("src.agents.repository_analysis_agent.nodes.github_client") as mock_client: + mock_client.get_file_content = AsyncMock(side_effect=Exception("API Error")) + mock_client.get_repository_contributors = AsyncMock(side_effect=Exception("API Error")) + + result = await agent.execute("test-owner/test-repo") + + assert isinstance(result, object) + + +class TestRuleRecommendation: + """Test cases for RuleRecommendation model.""" + + def test_valid_recommendation_creation(self): + """Test creating a valid rule recommendation.""" + rec = RuleRecommendation( + yaml_content="description: Test rule\nenabled: true", + confidence=0.8, + reasoning="Test reasoning", + source_patterns=["has_workflows"], + category="quality", + estimated_impact="high", + ) + + assert rec.yaml_content == "description: Test rule\nenabled: true" + assert rec.confidence == 0.8 + assert rec.category == "quality" + + def test_confidence_validation(self): + """Test confidence score validation.""" + # Valid confidence + rec = RuleRecommendation(yaml_content="test: rule", confidence=0.5, reasoning="test", category="test") + assert rec.confidence == 0.5 + + # Test bounds + with pytest.raises(ValueError): + RuleRecommendation(yaml_content="test: rule", confidence=1.5, reasoning="test", category="test") + + +class TestRepositoryAnalysisRequest: + """Test cases for RepositoryAnalysisRequest model.""" + + def test_valid_request(self): + """Test creating a valid analysis request.""" + request = RepositoryAnalysisRequest(repository_full_name="owner/repo", installation_id=12345) + + assert request.repository_full_name == "owner/repo" + assert request.installation_id == 12345 + + def test_request_without_installation_id(self): + """Test request without installation ID.""" + request = RepositoryAnalysisRequest(repository_full_name="owner/repo") + + assert request.repository_full_name == "owner/repo" + assert request.installation_id is None + + +class TestRepositoryFeatures: + """Test cases for RepositoryFeatures model.""" + + def test_features_initialization(self): + """Test repository features model.""" + features = RepositoryFeatures( + has_contributing=True, has_codeowners=True, has_workflows=True, contributor_count=10 + ) + + assert features.has_contributing is True + assert features.has_codeowners is True + assert features.has_workflows is True + assert features.contributor_count == 10 diff --git a/src/api/recommendations.py b/src/api/recommendations.py new file mode 100644 index 0000000..af9b495 --- /dev/null +++ b/src/api/recommendations.py @@ -0,0 +1,136 @@ +import logging + +from fastapi import APIRouter, HTTPException, Request +from fastapi.responses import JSONResponse + +from src.agents import get_agent +from src.agents.repository_analysis_agent.models import ( + RepositoryAnalysisRequest, + RepositoryAnalysisResponse, +) +from src.core.utils.caching import get_cache, set_cache +from src.core.utils.logging import log_structured + +router = APIRouter() +logger = logging.getLogger(__name__) + + +@router.post( + "/v1/rules/recommend", + response_model=RepositoryAnalysisResponse, + summary="Analyze repository and recommend rules", + description="Analyzes a GitHub repository and generates personalized Watchflow rule recommendations", +) +async def recommend_rules( + request: RepositoryAnalysisRequest, + req: Request, +) -> RepositoryAnalysisResponse: + """ + Analyze a repository and generate Watchflow rule recommendations. + + This endpoint analyzes the repository structure, contributing guidelines, + and patterns to recommend appropriate governance rules. + + Args: + request: Repository analysis request with repository identifier + req: FastAPI request object for logging + + Returns: + Repository analysis response with recommendations + + Raises: + HTTPException: If analysis fails or repository is invalid + """ + try: + if not request.repository_full_name or "/" not in request.repository_full_name: + raise HTTPException(status_code=400, detail="Invalid repository name format. Expected 'owner/repo'") + + cache_key = f"repo_analysis:{request.repository_full_name}" + cached_result = await get_cache(cache_key) + + if cached_result: + log_structured( + logger, + "cache_hit", + operation="repository_analysis", + subject_ids=[request.repository_full_name], + cached=True, + ) + return RepositoryAnalysisResponse(**cached_result) + + agent = get_agent("repository_analysis") + + log_structured( + logger, + "analysis_started", + operation="repository_analysis", + subject_ids=[request.repository_full_name], + installation_id=request.installation_id, + ) + + result = await agent.execute( + repository_full_name=request.repository_full_name, + installation_id=request.installation_id, + ) + + if not result.success: + log_structured( + logger, + "analysis_failed", + operation="repository_analysis", + subject_ids=[request.repository_full_name], + decision="failed", + error=result.message, + ) + raise HTTPException(status_code=500, detail=result.message) + + analysis_response = result.data.get("analysis_response") + if not analysis_response: + raise HTTPException(status_code=500, detail="No analysis response generated") + + await set_cache(cache_key, analysis_response.model_dump(), ttl=3600) + + log_structured( + logger, + "analysis_completed", + operation="repository_analysis", + subject_ids=[request.repository_full_name], + decision="success", + recommendations_count=len(analysis_response.recommendations), + latency_ms=result.metadata.get("execution_time_ms", 0), + ) + + return analysis_response + + except HTTPException as e: + raise e + except Exception as e: + logger.error(f"Error in recommend_rules endpoint: {e}") + log_structured( + logger, + "analysis_error", + operation="repository_analysis", + subject_ids=[request.repository_full_name] if request else [], + error=str(e), + ) + raise HTTPException(status_code=500, detail=f"Analysis failed: {str(e)}") from e + + +@router.get("/v1/rules/recommend/{repository_full_name}") +async def get_cached_recommendations(repository_full_name: str) -> JSONResponse: + """ + Get cached recommendations for a repository. + + Args: + repository_full_name: Full repository name (owner/repo) + + Returns: + Cached analysis results or 404 if not found + """ + cache_key = f"repo_analysis:{repository_full_name}" + cached_result = await get_cache(cache_key) + + if not cached_result: + raise HTTPException(status_code=404, detail="No cached analysis found for repository") + + return JSONResponse(content=cached_result) diff --git a/src/core/utils/caching.py b/src/core/utils/caching.py index 51313a9..6c0ce3a 100644 --- a/src/core/utils/caching.py +++ b/src/core/utils/caching.py @@ -115,6 +115,26 @@ def size(self) -> int: return len(self._cache) +# Simple module-level cache used by recommendations API +_GLOBAL_CACHE = AsyncCache(maxsize=1024, ttl=3600) + + +async def get_cache(key: str) -> Any | None: + """ + Async helper to fetch from the module-level cache. + """ + return _GLOBAL_CACHE.get(key) + + +async def set_cache(key: str, value: Any, ttl: int | None = None) -> None: + """ + Async helper to store into the module-level cache. + """ + if ttl and ttl != _GLOBAL_CACHE.ttl: + _GLOBAL_CACHE.ttl = ttl + _GLOBAL_CACHE.set(key, value) + + def cached_async( cache: AsyncCache | TTLCache | None = None, key_func: Callable[..., str] | None = None, diff --git a/src/core/utils/logging.py b/src/core/utils/logging.py index 65281e8..bcb77c9 100644 --- a/src/core/utils/logging.py +++ b/src/core/utils/logging.py @@ -7,6 +7,7 @@ import logging import time +from collections.abc import Callable from contextlib import asynccontextmanager from functools import wraps from typing import Any @@ -124,3 +125,22 @@ def sync_wrapper(*args, **kwargs): return sync_wrapper return decorator + + +def log_structured( + logger_obj: logging.Logger, + event: str, + level: str = "info", + **context: Any, +) -> None: + """ + Lightweight structured logging helper. + + Args: + logger_obj: Logger instance to use. + event: Event/operation name. + level: Logging level (info|warning|error). + **context: Arbitrary key/value metadata. + """ + log_fn: Callable[..., Any] = getattr(logger_obj, level, logger_obj.info) + log_fn(event, extra=context) diff --git a/src/event_processors/pull_request.py b/src/event_processors/pull_request.py index 8d16271..f7a4a66 100644 --- a/src/event_processors/pull_request.py +++ b/src/event_processors/pull_request.py @@ -221,6 +221,16 @@ async def _prepare_event_data_for_agent(self, task: Task, github_token: str) -> task.repo_full_name, pr_number, task.installation_id ) event_data["files"] = files or [] + event_data["changed_files"] = [ + { + "filename": file.get("filename"), + "status": file.get("status"), + "additions": file.get("additions"), + "deletions": file.get("deletions"), + } + for file in files or [] + ] + event_data["diff_summary"] = self._summarize_files_for_llm(files or []) except Exception as e: logger.warning(f"Error enriching event data: {e}") @@ -396,6 +406,41 @@ async def prepare_webhook_data(self, task: Task) -> dict[str, Any]: }, } + @staticmethod + def _summarize_files_for_llm(files: list[dict[str, Any]], max_files: int = 5, max_patch_lines: int = 8) -> str: + """ + Build a compact diff summary suitable for LLM prompts. + + Args: + files: GitHub file metadata objects (filename, status, additions, deletions, patch) + max_files: Max number of files to include in summary + max_patch_lines: Max patch lines per file (truncated beyond this) + + Returns: + Multiline summary string describing high-risk file changes with truncated patches. + """ + if not files: + return "" + + summary_lines: list[str] = [] + for file in files[:max_files]: + filename = file.get("filename", "unknown") + status = file.get("status", "modified") + additions = file.get("additions", 0) + deletions = file.get("deletions", 0) + summary_lines.append(f"- {filename} ({status}, +{additions}/-{deletions})") + + patch = file.get("patch") + if patch: + lines = patch.splitlines() + truncated = lines[:max_patch_lines] + indented_patch = "\n".join(f" {line}" for line in truncated) + summary_lines.append(indented_patch) + if len(lines) > max_patch_lines: + summary_lines.append(" ... (diff truncated)") + + return "\n".join(summary_lines) + async def prepare_api_data(self, task: Task) -> dict[str, Any]: """Fetch data not available in webhook.""" pr_data = task.payload.get("pull_request", {}) diff --git a/src/integrations/github/api.py b/src/integrations/github/api.py index 7e29db8..d899338 100644 --- a/src/integrations/github/api.py +++ b/src/integrations/github/api.py @@ -418,6 +418,43 @@ async def get_pull_request(self, repo: str, pr_number: int, installation_id: int logger.error(f"Error getting PR #{pr_number} from {repo}: {e}") return {} + async def list_pull_requests( + self, repo: str, installation_id: int, state: str = "all", per_page: int = 20 + ) -> list[dict[str, Any]]: + """ + List pull requests for a repository. + + Args: + repo: Full repo name (owner/repo) + installation_id: GitHub App installation id + state: "open", "closed", or "all" + per_page: max items to fetch (up to 100) + """ + try: + token = await self.get_installation_access_token(installation_id) + if not token: + logger.error(f"Failed to get installation token for {installation_id}") + return [] + + headers = {"Authorization": f"Bearer {token}", "Accept": "application/vnd.github.v3+json"} + url = f"{config.github.api_base_url}/repos/{repo}/pulls?state={state}&per_page={min(per_page, 100)}" + + session = await self._get_session() + async with session.get(url, headers=headers) as response: + if response.status == 200: + result = await response.json() + logger.info(f"Retrieved {len(result)} pull requests for {repo}") + return result + else: + error_text = await response.text() + logger.error( + f"Failed to list pull requests for {repo}. Status: {response.status}, Response: {error_text}" + ) + return [] + except Exception as e: + logger.error(f"Error listing pull requests for {repo}: {e}") + return [] + async def create_deployment_status( self, repo: str, diff --git a/src/main.py b/src/main.py index d3f96f3..7868196 100644 --- a/src/main.py +++ b/src/main.py @@ -5,6 +5,7 @@ from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware +from src.api.recommendations import router as recommendations_api_router from src.api.rules import router as rules_api_router from src.api.scheduler import router as scheduler_api_router from src.core.config import config @@ -102,6 +103,7 @@ async def lifespan(_app: FastAPI): app.include_router(webhook_router, prefix="/webhooks", tags=["GitHub Webhooks"]) app.include_router(rules_api_router, prefix="/api/v1", tags=["Public API"]) +app.include_router(recommendations_api_router, prefix="/api", tags=["Recommendations API"]) app.include_router(scheduler_api_router, prefix="/api/v1/scheduler", tags=["Scheduler API"]) # --- Root Endpoint --- diff --git a/src/rules/validators.py b/src/rules/validators.py index 33f7ca1..2d575c1 100644 --- a/src/rules/validators.py +++ b/src/rules/validators.py @@ -2,10 +2,82 @@ import re from abc import ABC, abstractmethod from datetime import datetime +from re import Pattern from typing import Any logger = logging.getLogger(__name__) +_GLOB_CACHE: dict[str, Pattern[str]] = {} + + +def _compile_glob(pattern: str) -> Pattern[str]: + """Convert a glob pattern supporting ** into a compiled regex.""" + cached = _GLOB_CACHE.get(pattern) + if cached: + return cached + + regex_parts: list[str] = [] + i = 0 + length = len(pattern) + while i < length: + char = pattern[i] + if char == "*": + if i + 1 < length and pattern[i + 1] == "*": + regex_parts.append(".*") + i += 1 + else: + regex_parts.append("[^/]*") + elif char == "?": + regex_parts.append("[^/]") + else: + regex_parts.append(re.escape(char)) + i += 1 + + compiled = re.compile("^" + "".join(regex_parts) + "$") + _GLOB_CACHE[pattern] = compiled + return compiled + + +def _expand_pattern_variants(pattern: str) -> set[str]: + """Generate fallback globs so ** can match zero directories.""" + variants = {pattern} + queue = [pattern] + + while queue: + current = queue.pop() + normalized = current.replace("//", "/") + + transformations = [ + ("/**/", "/"), + ("**/", ""), + ("/**", ""), + ("**", ""), + ] + + for old, new in transformations: + if old in normalized: + replaced = normalized.replace(old, new, 1) + replaced = replaced.replace("//", "/") + if replaced not in variants: + variants.add(replaced) + queue.append(replaced) + + return variants + + +def _matches_any(path: str, patterns: list[str]) -> bool: + """Utility matcher shared across validators.""" + if not path or not patterns: + return False + + normalized_path = path.replace("\\", "/") + for pattern in patterns: + for variant in _expand_pattern_variants(pattern.replace("\\", "/")): + compiled = _compile_glob(variant) + if compiled.match(normalized_path): + return True + return False + class Condition(ABC): """Abstract base class for all condition validators.""" @@ -738,6 +810,175 @@ def _is_new_contributor(self, username: str) -> bool: return True +class DiffPatternCondition(Condition): + """Validates that specific regex patterns appear (or do not appear) in PR diffs.""" + + name = "diff_pattern" + description = "Validates pull-request patches against required or forbidden regex patterns" + parameter_patterns = ["require_patterns", "forbidden_patterns", "file_patterns"] + event_types = ["pull_request"] + examples = [ + { + "file_patterns": ["packages/core/src/**/vector-query.ts"], + "require_patterns": ["throw\\s+new\\s+Error"], + }, + { + "file_patterns": ["packages/core/src/llm/**"], + "forbidden_patterns": ["console\\.log"], + }, + ] + + async def validate(self, parameters: dict[str, Any], event: dict[str, Any]) -> bool: + files = event.get("files", []) + if not files: + return True + + file_patterns = parameters.get("file_patterns") or ["**"] + require_patterns = parameters.get("require_patterns") or [] + forbidden_patterns = parameters.get("forbidden_patterns") or [] + + remaining_requirements = set(require_patterns) + + for file in files: + filename = file.get("filename", "") + if not filename or not _matches_any(filename, file_patterns): + continue + + patch = file.get("patch") + if not patch: + continue + + for pattern in list(remaining_requirements): + if re.search(pattern, patch, re.MULTILINE): + remaining_requirements.discard(pattern) + + for pattern in forbidden_patterns: + if re.search(pattern, patch, re.MULTILINE): + logger.debug( + "DiffPatternCondition: Forbidden pattern '%s' present in %s", + pattern, + filename, + ) + return False + + if remaining_requirements: + logger.debug( + "DiffPatternCondition: Required patterns missing -> %s", + remaining_requirements, + ) + return False + + return True + + +class RelatedTestsCondition(Condition): + """Ensures that changes to source files include corresponding test updates.""" + + name = "related_tests" + description = "Validates that touching core files requires touching tests" + parameter_patterns = ["source_patterns", "test_patterns", "min_test_files"] + event_types = ["pull_request"] + examples = [ + { + "source_patterns": ["packages/core/src/**"], + "test_patterns": ["packages/core/tests/**", "tests/**"], + } + ] + + async def validate(self, parameters: dict[str, Any], event: dict[str, Any]) -> bool: + files = event.get("files", []) + if not files: + return True + + source_patterns = parameters.get("source_patterns") or [] + test_patterns = parameters.get("test_patterns") or [] + min_test_files = parameters.get("min_test_files", 1) + + if not source_patterns or not test_patterns: + return True + + touched_sources = [ + file + for file in files + if file.get("status") != "removed" and _matches_any(file.get("filename", ""), source_patterns) + ] + + if not touched_sources: + return True + + touched_tests = [ + file + for file in files + if file.get("status") != "removed" and _matches_any(file.get("filename", ""), test_patterns) + ] + + is_valid = len(touched_tests) >= min_test_files + if not is_valid: + logger.debug( + "RelatedTestsCondition: %d source files touched but only %d test files updated", + len(touched_sources), + len(touched_tests), + ) + return is_valid + + +class RequiredFieldInDiffCondition(Condition): + """Validates that additions to specific files include a required field or text fragment.""" + + name = "required_field_in_diff" + description = "Ensures additions to matched files include specific text fragments" + parameter_patterns = ["file_patterns", "required_text"] + event_types = ["pull_request"] + examples = [ + { + "file_patterns": ["packages/core/src/agent/**/agent.py"], + "required_text": ["description:"], + } + ] + + async def validate(self, parameters: dict[str, Any], event: dict[str, Any]) -> bool: + files = event.get("files", []) + if not files: + return True + + file_patterns = parameters.get("file_patterns") or [] + required_text = parameters.get("required_text") + if not file_patterns or not required_text: + return True + + if isinstance(required_text, str): + required_text = [required_text] + + matched_files = False + + for file in files: + filename = file.get("filename", "") + if not filename or not _matches_any(filename, file_patterns): + continue + + patch = file.get("patch") + if not patch: + continue + + matched_files = True + additions = "\n".join( + line[1:] for line in patch.splitlines() if line.startswith("+") and not line.startswith("+++") + ) + + if all(text in additions for text in required_text): + return True + + # If we matched files but didn't find the required text, the rule fails. + if matched_files: + logger.debug( + "RequiredFieldInDiffCondition: Required text %s not present in additions", + required_text, + ) + return False + + return True + + # Registry of all available validators VALIDATOR_REGISTRY = { "author_team_is": AuthorTeamCondition(), @@ -763,6 +1004,9 @@ def _is_new_contributor(self, username: str) -> bool: "required_checks": RequiredChecksCondition(), "code_owners": CodeOwnersCondition(), "past_contributor_approval": PastContributorApprovalCondition(), + "diff_pattern": DiffPatternCondition(), + "related_tests": RelatedTestsCondition(), + "required_field_in_diff": RequiredFieldInDiffCondition(), } diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..1886775 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,12 @@ +""" +Pytest configuration to ensure the project root is on sys.path for imports. +""" + +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +SRC = ROOT / "src" + +if str(SRC) not in sys.path: + sys.path.insert(0, str(SRC)) diff --git a/tests/unit/event_processors/test_pull_request_processor.py b/tests/unit/event_processors/test_pull_request_processor.py new file mode 100644 index 0000000..5407d99 --- /dev/null +++ b/tests/unit/event_processors/test_pull_request_processor.py @@ -0,0 +1,26 @@ +from src.event_processors.pull_request import PullRequestProcessor + + +def test_summarize_files_for_llm_truncates_patch(): + files = [ + { + "filename": "packages/core/src/vector-query.ts", + "status": "modified", + "additions": 10, + "deletions": 2, + "patch": "+throw new Error('invalid filter')\n+return []\n+console.log('debug')", + } + ] + + summary = PullRequestProcessor._summarize_files_for_llm(files, max_files=1, max_patch_lines=2) + + assert "- packages/core/src/vector-query.ts (modified, +10/-2)" in summary + assert "throw new Error" in summary + assert "console.log" not in summary # truncated beyond max_patch_lines + assert "... (diff truncated)" in summary + + +def test_summarize_files_for_llm_handles_no_files(): + summary = PullRequestProcessor._summarize_files_for_llm([]) + + assert summary == "" diff --git a/tests/unit/rules/test_diff_validators.py b/tests/unit/rules/test_diff_validators.py new file mode 100644 index 0000000..2661902 --- /dev/null +++ b/tests/unit/rules/test_diff_validators.py @@ -0,0 +1,135 @@ +import pytest + +from src.rules.validators import ( + DiffPatternCondition, + RelatedTestsCondition, + RequiredFieldInDiffCondition, +) + + +@pytest.mark.asyncio +async def test_diff_pattern_condition_requirements_met(): + condition = DiffPatternCondition() + event = { + "files": [ + { + "filename": "packages/core/src/vector-query.ts", + "status": "modified", + "patch": "+throw new Error('invalid filter')\n+return []\n", + } + ] + } + + params = { + "file_patterns": ["packages/core/src/**/vector-query.ts"], + "require_patterns": ["throw\\s+new\\s+Error"], + } + + assert await condition.validate(params, event) + + +@pytest.mark.asyncio +async def test_diff_pattern_condition_missing_requirement(): + condition = DiffPatternCondition() + event = { + "files": [ + { + "filename": "packages/core/src/vector-query.ts", + "status": "modified", + "patch": "+return []\n", + } + ] + } + + params = { + "file_patterns": ["packages/core/src/**/vector-query.ts"], + "require_patterns": ["throw\\s+new\\s+Error"], + } + + assert not await condition.validate(params, event) + + +@pytest.mark.asyncio +async def test_related_tests_condition_requires_test_files(): + condition = RelatedTestsCondition() + event = { + "files": [ + { + "filename": "packages/core/src/vector-query.ts", + "status": "modified", + }, + { + "filename": "tests/vector-query.test.ts", + "status": "modified", + }, + ] + } + + params = { + "source_patterns": ["packages/core/src/**"], + "test_patterns": ["tests/**"], + } + + assert await condition.validate(params, event) + + +@pytest.mark.asyncio +async def test_related_tests_condition_flags_missing_tests(): + condition = RelatedTestsCondition() + event = { + "files": [ + { + "filename": "packages/core/src/vector-query.ts", + "status": "modified", + } + ] + } + + params = { + "source_patterns": ["packages/core/src/**"], + "test_patterns": ["tests/**"], + } + + assert not await condition.validate(params, event) + + +@pytest.mark.asyncio +async def test_required_field_in_diff_condition(): + condition = RequiredFieldInDiffCondition() + event = { + "files": [ + { + "filename": "packages/core/src/agent/foo/agent.py", + "status": "modified", + "patch": '+class FooAgent:\n+ description = "foo"\n', + } + ] + } + + params = { + "file_patterns": ["packages/core/src/agent/**"], + "required_text": "description", + } + + assert await condition.validate(params, event) + + +@pytest.mark.asyncio +async def test_required_field_in_diff_condition_missing_text(): + condition = RequiredFieldInDiffCondition() + event = { + "files": [ + { + "filename": "packages/core/src/agent/foo/agent.py", + "status": "modified", + "patch": "+class FooAgent:\n+ pass\n", + } + ] + } + + params = { + "file_patterns": ["packages/core/src/agent/**"], + "required_text": "description", + } + + assert not await condition.validate(params, event) diff --git a/tests/unit/test_mastra_rules_sample.py b/tests/unit/test_mastra_rules_sample.py new file mode 100644 index 0000000..116a911 --- /dev/null +++ b/tests/unit/test_mastra_rules_sample.py @@ -0,0 +1,23 @@ +"""Regression test for the Mastra sample rules.""" + +from pathlib import Path + +import yaml + +from src.rules.models import Rule + +SAMPLE_RULES_PATH = Path(__file__).resolve().parents[2] / "docs" / "assets" / "mastra-watchflow-rules.yaml" + + +def test_mastra_sample_rules_validate_without_actions(): + """Ensure the Mastra sample rules stay compatible with the current rule schema.""" + assert SAMPLE_RULES_PATH.exists(), "Sample rules file is missing" + + data = yaml.safe_load(SAMPLE_RULES_PATH.read_text()) + assert isinstance(data, dict) and "rules" in data, "Sample file must include a top-level 'rules' list" + + for rule in data["rules"]: + validated_rule = Rule.model_validate(rule) + # Loader stores actions but invocation pipeline currently ignores them. + # Keep the sample intentionally simple until action semantics are implemented. + assert not validated_rule.actions, "Sample rules must omit 'actions' entries"