Testing & QA Framework

"""
Unit tests for LeagueAnalysisService using Pydantic schemas
from data-layer-registry for type-safe testing.
"""
import pytest
from unittest.mock import AsyncMock, patch
from datetime import datetime
 
# Import schemas from data-layer-registry
from apps.backend.schemas.league import (
    LeagueSchema,
    LeagueAnalysisRequest,
    LeagueAnalysisResponse,
    PartnershipScoreSchema,
    TierRecommendationSchema
)
from apps.backend.schemas.scoring import (
    ScoringDimensionSchema,
    ScoreBreakdownSchema
)
from apps.backend.services.league_analysis_service import LeagueAnalysisService
from apps.backend.tests.fixtures.league_fixtures import (
    league_factory,
    analysis_request_factory,
    mock_ai_client
)
 
 
class TestLeagueAnalysisService:
    """Test suite for League Analysis Service"""
 
    @pytest.fixture
    def service(self, mock_ai_client):
        """Create service instance with mocked AI client"""
        return LeagueAnalysisService(ai_client=mock_ai_client)
 
    @pytest.fixture
    def sample_league(self):
        """Generate sample league using factory with schema validation"""
        return league_factory.create(
            name="Premier Lacrosse League",
            sport="Lacrosse",
            region="North America",
            tier=None  # To be determined by analysis
        )
 
    @pytest.mark.asyncio
    async def test_analyze_league_success(self, service, sample_league):
        """
        GIVEN: A valid league object conforming to LeagueSchema
        WHEN: analyze_league() is called
        THEN: Returns LeagueAnalysisResponse with tier recommendation
        """
        # Create request using schema
        request = LeagueAnalysisRequest(
            league_name=sample_league.name,
            website_url="https://premierlacrosseleague.com",
            contact_email="partnerships@pll.com"
        )
 
        # Execute analysis
        response = await service.analyze_league(request)
 
        # Validate response schema
        assert isinstance(response, LeagueAnalysisResponse)
        assert response.league_name == sample_league.name
        assert response.recommended_tier in ["1.1", "1.2", "2.1", "2.2", "3.1", "3.2", "4.1", "4.2"]
        assert 0 <= response.partnership_score.total_score <= 100
 
        # Validate scoring dimensions
        assert len(response.partnership_score.dimensions) == 7
        dimension_names = {d.dimension_name for d in response.partnership_score.dimensions}
        expected_dimensions = {
            "market_size", "data_availability", "betting_popularity",
            "growth_potential", "technical_feasibility",
            "partnership_readiness", "revenue_potential"
        }
        assert dimension_names == expected_dimensions
 
    @pytest.mark.asyncio
    async def test_analyze_league_invalid_data(self, service):
        """
        GIVEN: Invalid league data (fails schema validation)
        WHEN: analyze_league() is called
        THEN: Raises ValidationError with detailed message
        """
        from pydantic import ValidationError
 
        with pytest.raises(ValidationError) as exc_info:
            request = LeagueAnalysisRequest(
                league_name="",  # Invalid: empty string
                website_url="not-a-url",  # Invalid URL format
                contact_email="invalid-email"  # Invalid email
            )
 
        errors = exc_info.value.errors()
        assert len(errors) >= 3
 
    @pytest.mark.asyncio
    async def test_scoring_breakdown_calculation(self, service, sample_league):
        """
        GIVEN: A league with known characteristics
        WHEN: Partnership score is calculated
        THEN: Each dimension score is weighted correctly
        """
        request = LeagueAnalysisRequest(
            league_name=sample_league.name,
            website_url="https://example.com",
            contact_email="test@example.com"
        )
 
        with patch.object(service, '_fetch_market_data') as mock_market:
            # Mock market data with known values
            mock_market.return_value = {
                "market_size": 85,
                "data_availability": 70,
                "betting_popularity": 60
            }
 
            response = await service.analyze_league(request)
 
            # Verify weighted calculation
            breakdown = response.partnership_score
            total = sum(d.score * d.weight for d in breakdown.dimensions)
            expected_total = total / sum(d.weight for d in breakdown.dimensions)
 
            assert abs(breakdown.total_score - expected_total) < 0.01

[pytest]
testpaths = tests
python_files = test_*.py
python_classes = Test*
python_functions = test_*
asyncio_mode = auto
markers =
    unit: Unit tests (fast, isolated)
    integration: Integration tests (slower, requires services)
    slow: Slow tests (>1s execution time)
    ai_persona: AI persona evaluation tests
    security: Security-focused tests
addopts =
    --strict-markers
    --cov=apps/backend
    --cov-report=html
    --cov-report=term-missing
    --cov-fail-under=80
    --maxfail=5
    -v

# apps/backend/tests/integration/test_league_integration.py
import pytest
from httpx import AsyncClient
from apps.backend.main import app
from apps.backend.tests.fixtures.database_fixtures import (
    create_test_database,
    truncate_tables
)
 
@pytest.mark.asyncio
async def test_league_creation_workflow():
    """
    Integration test for complete league creation workflow
    Tests: API endpoint -> Database persistence -> Schema validation
    """
    # Setup test database
    test_db = await create_test_database()
    
    # Create test client
    async with AsyncClient(app=app, base_url="http://test") as ac:
        # Test 1: Create league via API
        response = await ac.post(
            "/api/leagues",
            json={
                "name": "Test League Integration",
                "sport": "Basketball",
                "region": "North America",
                "contact_email": "test@league.com"
            },
            headers={"Authorization": "Bearer test_token"}
        )
        
        assert response.status_code == 201
        created_league = response.json()
        assert created_league["name"] == "Test League Integration"
        assert "id" in created_league
        
        # Test 2: Verify database persistence
        league_id = created_league["id"]
        db_league = await test_db.fetchrow(
            "SELECT * FROM leagues WHERE id = $1",
            (league_id,)
        )
        assert db_league["name"] == "Test League Integration"
        assert db_league["sport"] == "Basketball"
        
        # Test 3: Update league via API
        update_response = await ac.put(
            f"/api/leagues/{league_id}",
            json={
                "name": "Updated League Integration",
                "sport": "Basketball",
                "region": "Europe"
            },
            headers={"Authorization": "Bearer test_token"}
        )
        
        assert update_response.status_code == 200
        updated_league = update_response.json()
        assert updated_league["name"] == "Updated League Integration"
        assert updated_league["region"] == "Europe"
        
        # Test 4: Verify database update
        db_updated = await test_db.fetchrow(
            "SELECT * FROM leagues WHERE id = $1",
            (league_id,)
        )
        assert db_updated["name"] == "Updated League Integration"
        assert db_updated["region"] == "Europe"
        
        # Test 5: Delete league via API
        delete_response = await ac.delete(
            f"/api/leagues/{league_id}",
            headers={"Authorization": "Bearer test_token"}
        )
        
        assert delete_response.status_code == 200
        
        # Test 6: Verify database deletion
        db_deleted = await test_db.fetchrow(
            "SELECT * FROM leagues WHERE id = $1",
            (league_id,)
        )
        assert db_deleted is None
 
# Cleanup - transaction rollback handled by fixture

// clients/frontend/e2e/league-onboarding.spec.ts
import { test, expect } from '@playwright/test';
import { LeagueOnboardingPage } from './pages/LeagueOnboardingPage';
import { DashboardPage } from './pages/DashboardPage';
 
test.describe('League Onboarding E2E Flow', () => {
  test('Complete league onboarding journey', async ({ page }) => {
    const onboardingPage = new LeagueOnboardingPage(page);
    const dashboardPage = new DashboardPage(page);
 
    // Step 1: Navigate to registration
    await page.goto('/register');
    await expect(page).toHaveTitle(/Register/);
 
    // Step 2: Fill registration form
    await onboardingPage.fillRegistrationForm({
      email: 'testuser@example.com',
      password: 'SecurePass123!',
      leagueName: 'Test Integration League',
      sportDomain: 'team_player_actions',
      targetSportsbooks: [
        { id: 'draftkings', priority: 1 },
        { id: 'underdog', priority: 2 }
      ]
    });
 
    // Step 3: Complete registration
    await onboardingPage.submitRegistration();
    await expect(page).toHaveURL(/dashboard/);
 
    // Step 4: Verify dashboard
    await dashboardPage.verifyLeagueCreated('Test Integration League');
    await expect(dashboardPage.leagueCard).toBeVisible();
 
    // Step 5: Upload document
    await dashboardPage.uploadDocument('test-questionnaire.pdf');
    await expect(dashboardPage.successMessage).toBeVisible({ text: /uploaded/ });
 
    // Step 6: Verify compliance status
    await dashboardPage.verifyComplianceStatus('Test Integration League');
    await expect(dashboardPage.complianceScore).toBeVisible();
 
    // Step 7: Generate contract
    await dashboardPage.generateContract();
    await expect(dashboardPage.contractStatus).toBeVisible({ text: /generated/ });
  });
 
  test('Handle registration error scenarios', async ({ page }) => {
    const onboardingPage = new LeagueOnboardingPage(page);
 
    await page.goto('/register');
 
    // Test invalid email
    await onboardingPage.fillRegistrationForm({
      email: 'invalid-email',
      password: 'SecurePass123!',
      leagueName: 'Test League'
    });
    await onboardingPage.submitRegistration();
    await expect(onboardingPage.errorMessage).toBeVisible({ text: /email/ });
 
    // Test weak password
    await onboardingPage.fillRegistrationForm({
      email: 'valid@example.com',
      password: 'weak',
      leagueName: 'Test League'
    });
    await onboardingPage.submitRegistration();
    await expect(onboardingPage.errorMessage).toBeVisible({ text: /password/ });
  });
});

"""
Base AI Persona class that uses schemas for structured evaluation.
All persona evaluations return validated SchematicEvaluationResult.
"""
from abc import ABC, abstractmethod
from typing import Dict, List, Any, Optional
from pydantic import BaseModel, Field, validator
from openai import AsyncOpenAI
from anthropic import AsyncAnthropic
 
# Import evaluation schemas from data-layer-registry
from apps.backend.schemas.evaluation import (
    EvaluationResultSchema,
    SatisfactionScoreSchema,
    IssueReportSchema,
    PersonaFeedbackSchema
)
 
 
class AIPersona(ABC):
    """Base class for AI testing personas"""
 
    def __init__(
        self,
        persona_name: str,
        persona_description: str,
        expertise_areas: List[str],
        ai_client: Optional[AsyncOpenAI | AsyncAnthropic] = None
    ):
        self.persona_name = persona_name
        self.persona_description = persona_description
        self.expertise_areas = expertise_areas
        self.ai_client = ai_client or AsyncOpenAI()
 
    @abstractmethod
    async def evaluate_feature(
        self,
        feature_name: str,
        feature_data: Dict[str, Any],
        evaluation_criteria: List[str]
    ) -> EvaluationResultSchema:
        """
        Evaluate a feature from persona's perspective.
        Returns validated EvaluationResultSchema from data-layer-registry.
        """
        pass
 
    async def _generate_evaluation(
        self,
        prompt: str,
        context: Dict[str, Any]
    ) -> PersonaFeedbackSchema:
        """Generate persona feedback using LLM"""
        system_prompt = f"""
        You are {self.persona_name}: {self.persona_description}
 
        Your expertise areas: {', '.join(self.expertise_areas)}
 
        Evaluate the feature and provide structured feedback in JSON format
        matching the PersonaFeedbackSchema:
        {{
            "satisfaction_score": 0-100,
            "strengths": ["list of strengths"],
            "weaknesses": ["list of weaknesses"],
            "suggestions": ["list of improvement suggestions"],
            "critical_issues": [
                {{
                    "severity": "critical|high|medium|low",
                    "category": "usability|performance|security|functionality",
                    "description": "issue description",
                    "suggested_fix": "how to fix"
                }}
            ],
            "overall_impression": "detailed feedback"
        }}
        """
        response = await self.ai_client.chat.completions.create(
            model="gpt-4-turbo-preview",
            messages=[
                {"role": "system", "content": system_prompt},
                {"role": "user", "content": prompt}
            ],
            response_format={"type": "json_object"},
            temperature=0.7
        )
 
        feedback_data = json.loads(response.choices[0].message.content)
 
        # Validate against schema
        return PersonaFeedbackSchema(**feedback_data)
 
 
class LeagueAdminPersona(AIPersona):
    """
    Persona: League Administrator
    Perspective: Non-technical league operations manager
    Focus: Ease of use, onboarding, partnership process
    """
    def __init__(self):
        super().__init__(
            persona_name="League Administrator",
            persona_description="A non-technical operations manager for a mid-sized sports league",
            expertise_areas=[
                "League operations",
                "Partnership management",
                "Document processing",
                "Non-technical workflows"
            ]
        )
 
    async def evaluate_feature(
        self,
        feature_name: str,
        feature_data: Dict[str, Any],
        evaluation_criteria: List[str]
    ) -> EvaluationResultSchema:
        """Evaluate from league admin perspective"""
 
        prompt = f"""
        Evaluate the '{feature_name}' feature as a league administrator.
 
        Feature Details:
        {json.dumps(feature_data, indent=2)}
 
        Evaluation Criteria:
        {chr(10).join(f"- {criterion}" for criterion in evaluation_criteria)}
 
        Focus on:
        1. How easy is it for non-technical users?
        2. Is the onboarding process clear?
        3. Can I complete tasks without technical help?
        4. Are error messages understandable?
        5. Is the partnership process transparent?
        """
        feedback = await self._generate_evaluation(prompt, feature_data)
 
        # Convert to full evaluation result
        return EvaluationResultSchema(
            persona_name=self.persona_name,
            feature_name=feature_name,
            satisfaction_score=SatisfactionScoreSchema(
                overall=feedback.satisfaction_score,
                usability=feedback.satisfaction_score,  # League admin focuses on usability
                functionality=feedback.satisfaction_score * 0.9,
                performance=feedback.satisfaction_score * 0.8
            ),
            strengths=feedback.strengths,
            weaknesses=feedback.weaknesses,
            suggestions=feedback.suggestions,
            critical_issues=feedback.critical_issues,
            overall_impression=feedback.overall_impression,
            timestamp=datetime.utcnow().isoformat()
        )
 
 
class TechEvaluatorPersona(AIPersona):
    """
    Persona: Technical Evaluator
    Perspective: Senior software engineer
    Focus: Code quality, architecture, performance, security
    """
    def __init__(self):
        super().__init__(
            persona_name="Technical Evaluator",
            persona_description="A senior software engineer with 10+ years experience",
            expertise_areas=[
                "Software architecture",
                "Code quality",
                "Performance optimization",
                "Security best practices",
                "API design"
            ]
        )
 
    async def evaluate_feature(
        self,
        feature_name: str,
        feature_data: Dict[str, Any],
        evaluation_criteria: List[str]
    ) -> EvaluationResultSchema:
        """Evaluate from technical perspective"""
 
        prompt = f"""
        Evaluate the '{feature_name}' feature as a senior software engineer.
 
        Feature Details:
        {json.dumps(feature_data, indent=2)}
 
        Evaluation Criteria:
        {chr(10).join(f"- {criterion}" for criterion in evaluation_criteria)}
 
        Focus on:
        1. Is the code well-architected and maintainable?
        2. Are there performance bottlenecks?
        3. Are security best practices followed?
        4. Is error handling comprehensive?
        5. Is the API design RESTful and intuitive?
        6. Are schemas properly validated?
        7. Is the code testable?
        """
        feedback = await self._generate_evaluation(prompt, feature_data)
 
        return EvaluationResultSchema(
            persona_name=self.persona_name,
            feature_name=feature_name,
            satisfaction_score=SatisfactionScoreSchema(
                overall=feedback.satisfaction_score,
                usability=feedback.satisfaction_score * 0.7,
                functionality=feedback.satisfaction_score,
                performance=feedback.satisfaction_score * 0.95
            ),
            strengths=feedback.strengths,
            weaknesses=feedback.weaknesses,
            suggestions=feedback.suggestions,
            critical_issues=feedback.critical_issues,
            overall_impression=feedback.overall_impression,
            timestamp=datetime.utcnow().isoformat()
        )

"""
Orchestrates parallel execution of 6 AI persona evaluations.
Aggregates results into comprehensive quality report.
"""
import asyncio
from typing import List, Dict, Any
from concurrent.futures import ThreadPoolExecutor
 
from apps.backend.schemas.evaluation import (
    EvaluationResultSchema,
    AggregatedEvaluationSchema,
    QualityReportSchema
)
from tests.ai_personas.personas import (
    LeagueAdminPersona,
    TechEvaluatorPersona,
    BusinessPartnerPersona,
    SportsBettingExpertPersona,
    DataScientistPersona,
    NewcomerPersona
)
 
 
class PersonaEvaluationOrchestrator:
    """Orchestrates 6-persona AI evaluation"""
 
    def __init__(self):
        self.personas = [
            LeagueAdminPersona(),
            TechEvaluatorPersona(),
            BusinessPartnerPersona(),
            SportsBettingExpertPersona(),
            DataScientistPersona(),
            NewcomerPersona()
        ]
 
    async def evaluate_system(
        self,
        features: List[Dict[str, Any]],
        evaluation_criteria: List[str]
    ) -> QualityReportSchema:
        """
        Run comprehensive 6-persona evaluation in parallel.
        Returns aggregated QualityReportSchema.
        """
        all_evaluations: List[EvaluationResultSchema] = []
 
        # Run all persona evaluations in parallel
        tasks = []
        for feature in features:
            for persona in self.personas:
                task = persona.evaluate_feature(
                    feature_name=feature['name'],
                    feature_data=feature['data'],
                    evaluation_criteria=evaluation_criteria
                )
                tasks.append(task)
 
        # Execute in parallel with timeout
        evaluations = await asyncio.gather(*tasks, return_exceptions=True)
 
        # Filter out exceptions and collect valid evaluations
        for eval_result in evaluations:
            if isinstance(eval_result, EvaluationResultSchema):
                all_evaluations.append(eval_result)
            else:
                # Log exception
                print(f"Evaluation failed: {eval_result}")
 
        # Aggregate results
        return self._aggregate_evaluations(all_evaluations)
 
    def _aggregate_evaluations(
        self,
        evaluations: List[EvaluationResultSchema]
    ) -> QualityReportSchema:
        """Aggregate persona evaluations into quality report"""
 
        # Calculate average satisfaction scores
        avg_overall = sum(e.satisfaction_score.overall for e in evaluations) / len(evaluations)
        avg_usability = sum(e.satisfaction_score.usability for e in evaluations) / len(evaluations)
        avg_functionality = sum(e.satisfaction_score.functionality for e in evaluations) / len(evaluations)
        avg_performance = sum(e.satisfaction_score.performance for e in evaluations) / len(evaluations)
 
        # Collect all critical issues
        all_issues = []
        for evaluation in evaluations:
            all_issues.extend(evaluation.critical_issues)
 
        # Group issues by severity
        critical_issues = [i for i in all_issues if i.severity == "critical"]
        high_issues = [i for i in all_issues if i.severity == "high"]
 
        # Generate quality grade
        quality_grade = self._calculate_quality_grade(
            avg_overall,
            len(critical_issues),
            len(high_issues)
        )
 
        return QualityReportSchema(
            report_id=str(uuid.uuid4()),
            average_satisfaction=avg_overall,
            satisfaction_breakdown={
                "usability": avg_usability,
                "functionality": avg_functionality,
                "performance": avg_performance
            },
            total_evaluations=len(evaluations),
            personas_evaluated=[e.persona_name for e in evaluations],
            critical_issues_count=len(critical_issues),
            high_issues_count=len(high_issues),
            quality_grade=quality_grade,
            recommendations=self._generate_recommendations(evaluations),
            detailed_evaluations=evaluations,
            generated_at=datetime.utcnow().isoformat()
        )
 
    def _calculate_quality_grade(
        self,
        avg_score: float,
        critical_issues: int,
        high_issues: int
    ) -> str:
        """Calculate quality grade: A+, A, B+, B, C, D, F"""
 
        if critical_issues > 0:
            return "D" if critical_issues <= 2 else "F"
 
        if high_issues > 5:
            return "C"
 
        if avg_score >= 90:
            return "A+" if high_issues == 0 else "A"
        elif avg_score >= 80:
            return "B+" if high_issues <= 2 else "B"
        elif avg_score >= 70:
            return "C"
        else:
            return "D"

"""
Performance testing using Locust.
Tests backend API endpoints with realistic load patterns.
"""
from locust import HttpUser, task, between
import random
from datetime import datetime
 
# Import request/response schemas for type-safe testing
from apps.backend.schemas.league import LeagueAnalysisRequest
from apps.backend.schemas.document import DocumentUploadRequest
 
 
class LeagueManagementUser(HttpUser):
    """Simulates typical user behavior"""
 
    wait_time = between(1, 5)  # 1-5 seconds between requests
 
    def on_start(self):
        """Login and setup"""
        response = self.client.post("/api/auth/login", json={
            "email": "test@example.com",
            "password": "testpass123"
        })
        self.token = response.json()["access_token"]
        self.headers = {"Authorization": f"Bearer {self.token}"}
 
    @task(3)  # 30% of requests
    def view_dashboard(self):
        """Load league dashboard"""
        self.client.get("/api/leagues", headers=self.headers)
 
    @task(2)  # 20% of requests
    def analyze_league(self):
        """Trigger league analysis"""
        request = LeagueAnalysisRequest(
            league_name=f"Test League {random.randint(1, 100)}",
            website_url="https://example.com",
            contact_email="test@example.com"
        )
 
        with self.client.post(
            "/api/leagues/analyze",
            json=request.dict(),
            headers=self.headers,
            catch_response=True
        ) as response:
            if response.status_code == 200:
                # Verify response matches schema
                data = response.json()
                if "recommended_tier" in data and "partnership_score" in data:
                    response.success()
                else:
                    response.failure("Invalid response schema")
            else:
                response.failure(f"Failed with status {response.status_code}")
 
    @task(1)  # 10% of requests
    def upload_document(self):
        """Upload league questionnaire"""
        files = {
            "file": ("questionnaire.pdf", b"fake pdf content", "application/pdf")
        }
 
        self.client.post(
            "/api/documents/upload",
            files=files,
            headers=self.headers
        )
 
    @task(1)  # 10% of requests
    def search_leagues(self):
        """Search for leagues"""
        query = random.choice(["basketball", "football", "lacrosse", "hockey"])
        self.client.get(
            f"/api/leagues/search?q={query}",
            headers=self.headers
        )

# security/vulnerability/vulnerability_scanner.py
import subprocess
import json
from typing import Dict, List, Any
from datetime import datetime
 
class VulnerabilityScanner:
    """Automated vulnerability scanning"""
 
    def __init__(self, project_root: str):
        self.project_root = project_root
        self.scan_results = []
 
    async def scan_code_sast(self, target: str = "all") -> List[SecurityFinding]:
        """Run SAST code scanning"""
 
        findings = []
 
        # Python code (Bandit)
        if target in ["all", "python"]:
            try:
                result = subprocess.run(
                    ["bandit", "-r", f"{self.project_root}/apps/backend", "-f", "json", "-o", "-"],
                    capture_output=True,
                    text=True,
                    check=True
                )
                python_findings = json.loads(result.stdout)
                for issue in python_findings.get("results", []):
                    findings.append(SecurityFinding(
                        tool="bandit",
                        severity=issue["issue_severity"],
                        confidence=issue["issue_confidence"],
                        description=issue["issue_text"],
                        location=f"{self.project_root}/apps/backend/{issue['filename']}:{issue['line_number']}",
                        timestamp=datetime.utcnow()
                    ))
            except subprocess.CalledProcessError as e:
                logger.error("Bandit scan failed", error=str(e))
 
        # JavaScript/TypeScript (ESLint with security plugins)
        if target in ["all", "frontend"]:
            try:
                result = subprocess.run(
                    [
                        "npx", "eslint", "--ext", ".js,.ts,.tsx", 
                        "--format", "json",
                        f"{self.project_root}/clients/frontend/src"
                    ],
                    capture_output=True,
                    text=True,
                    check=True
                )
                js_findings = json.loads(result.stdout)
                for file_results in js_findings:
                    for issue in file_results["messages"]:
                        if issue["ruleId"] in ["security/detect-unsafe-regex", "security/detect-possible-timing-attacks"]:
                            findings.append(SecurityFinding(
                                tool="eslint-security",
                                severity=issue["severity"][0].upper(),
                                confidence="high",
                                description=issue["message"],
                                location=f"{self.project_root}/clients/frontend/src/{file_results['filePath']}:{issue['line']}",
                                timestamp=datetime.utcnow()
                            ))
            except subprocess.CalledProcessError as e:
                logger.error("ESLint security scan failed", error=str(e))
 
        return findings
 
    async def scan_dependencies(self) -> List[SecurityFinding]:
        """Scan dependencies for vulnerabilities"""
 
        findings = []
 
        # Python dependencies (Safety)
        try:
            result = subprocess.run(
                ["safety", "check", "--json"],
                capture_output=True,
                text=True,
                cwd=self.project_root,
                check=True
            )
            vuln_data = json.loads(result.stdout)
            for vuln in vuln_data:
                findings.append(SecurityFinding(
                    tool="safety",
                    severity=vuln["package"]["advisories"][0]["advisory"]["severity"],
                    confidence="high",
                    description=f"Dependency vulnerability: {vuln['package']['name']} {vuln['package']['version']} - {vuln['package']['advisories'][0]['advisory']['description']}",
                    location=f"{self.project_root}/pyproject.toml",
                    timestamp=datetime.utcnow()
                ))
        except subprocess.CalledProcessError as e:
            logger.error("Safety scan failed", error=str(e))
 
        # JavaScript dependencies (npm audit)
        try:
            result = subprocess.run(
                ["npm", "audit", "--json"],
                capture_output=True,
                text=True,
                cwd=f"{self.project_root}/clients/frontend",
                check=True
            )
            audit_data = json.loads(result.stdout)
            for vuln in audit_data.get("vulnerabilities", {}).values():
                if vuln["severity"] in ["high", "critical"]:
                    findings.append(SecurityFinding(
                        tool="npm-audit",
                        severity=vuln["severity"].upper(),
                        confidence="high",
                        description=f"Dependency vulnerability: {vuln['name']}@{vuln['version']} - {vuln['overview']}",
                        location=f"{self.project_root}/clients/frontend/package.json",
                        timestamp=datetime.utcnow()
                    ))
        except subprocess.CalledProcessError as e:
            logger.error("npm audit failed", error=str(e))
 
        return findings
 
    async def scan_containers(self) -> List[SecurityFinding]:
        """Scan Docker images for vulnerabilities"""
 
        findings = []
 
        # Run Trivy scan on all images
        try:
            result = subprocess.run(
                ["trivy", "image", "--format", "json", "--exit-code", "0", "--no-progress"],
                capture_output=True,
                text=True,
                cwd=self.project_root,
                check=True
            )
            trivy_results = json.loads(result.stdout)
            
            for result in trivy_results.get("Results", []):
                for vuln in result.get("Vulnerabilities", []):
                    if vuln["Severity"] in ["CRITICAL", "HIGH"]:
                        findings.append(SecurityFinding(
                            tool="trivy",
                            severity=vuln["Severity"],
                            confidence="high",
                            description=f"Container vulnerability: {vuln['VulnerabilityID']} - {vuln['Title']}",
                            location=f"Docker image: {result['Target']}",
                            timestamp=datetime.utcnow()
                        ))
        except subprocess.CalledProcessError as e:
            logger.error("Trivy scan failed", error=str(e))
 
        return findings
 
    async def run_owasp_zap_scan(self, target_url: str) -> List[SecurityFinding]:
        """Run OWASP ZAP dynamic analysis"""
 
        findings = []
 
        # Configure and run ZAP scan
        try:
            # Start ZAP in daemon mode (in production, use containerized ZAP)
            zap_config = {
                "target": target_url,
                "ajaxSpider": True,
                "context": "Production API",
                "attackStrength": "MEDIUM",
                "alertThreshold": "HIGH"
            }
 
            # Execute scan (simplified - in practice, use ZAP API)
            # result = await self._execute_zap_scan(zap_config)
 
            # Parse results
            # for alert in result.alerts:
            #     if alert.risk in ["High", "Medium"]:
            #         findings.append(SecurityFinding(...))
 
            pass  # Placeholder for ZAP integration
 
        except Exception as e:
            logger.error("OWASP ZAP scan failed", error=str(e))
 
        return findings
 
    async def generate_vulnerability_report(self, scan_results: List[SecurityFinding]) -> VulnerabilityReport:
        """Generate comprehensive vulnerability report"""
 
        # Categorize findings
        critical = [f for f in scan_results if f.severity == "CRITICAL"]
        high = [f for f in scan_results if f.severity == "HIGH"]
        medium = [f for f in scan_results if f.severity == "MEDIUM"]
        low = [f for f in scan_results if f.severity == "LOW"]
 
        # Generate remediation plan
        remediation_plan = await self._generate_remediation_plan(scan_results)
 
        return VulnerabilityReport(
            total_findings=len(scan_results),
            critical_count=len(critical),
            high_count=len(high),
            medium_count=len(medium),
            low_count=len(low),
            findings=scan_results,
            remediation_plan=remediation_plan,
            report_date=datetime.utcnow(),
            priority="immediate" if critical else "high"
        )

# apps/backend/tests/fixtures/league_fixtures.py
import factory
from factory import fuzzy
from faker import Faker
from datetime import datetime, timedelta
from uuid import uuid4
 
from apps.backend.models.league import League
from apps.backend.models.user import User
from apps.backend.tests.fixtures.user_fixtures import user_factory
 
fake = Faker()
 
class LeagueFactory(factory.Factory):
    """Factory for League model with schema validation"""
 
    class Meta:
        model = League
 
    id = factory.LazyAttribute(lambda _: str(uuid4()))
    name = fuzzy.FuzzyText(length=20)
    sport = fuzzy.FuzzyChoice(choices=["Basketball", "Football", "Lacrosse", "Hockey"])
    region = fuzzy.FuzzyChoice(choices=["North America", "Europe", "Asia"])
    tier = fuzzy.FuzzyChoice(choices=["1.1", "1.2", "2.1", "2.2", "3.1", "3.2", "4.1", "4.2"])
    partnership_score = fuzzy.FuzzyFloat(low=0.0, high=100.0)
    created_at = factory.LazyAttribute(lambda _: datetime.utcnow())
    updated_at = factory.LazyAttribute(lambda _: datetime.utcnow())
 
    owner = factory.SubFactory(user_factory)
 
    @factory.post_generation
    def post_generation(self, create, extracted, **kwargs):
        """Post-generation hooks for complex relationships"""
        if create:
            # Ensure owner exists
            if not self.owner:
                self.owner = user_factory.create()
            
            # Validate tier format
            if self.tier not in ["1.1", "1.2", "2.1", "2.2", "3.1", "3.2", "4.1", "4.2"]:
                self.tier = "2.1"  # Default valid tier
 
    @classmethod
    def create_league_with_partnerships(cls, count: int = 3):
        """Create league with associated partnerships"""
        league = cls.create()
        
        # Create partnerships
        for _ in range(count):
            partnership = PartnershipFactory.create(league=league)
        
        return league
 
    @classmethod
    def create_realistic_league(cls, sport_domain: str = "team_player_actions"):
        """Create realistic league for given sport domain"""
        sport_choices = {
            "combat": ["Boxing", "MMA", "Wrestling"],
            "racing": ["Horse Racing", "Auto Racing", "Motorcycle Racing"],
            "action_sports": ["Skateboarding", "BMX", "Snowboarding"],
            "team_player_actions": ["Basketball", "Football", "Hockey"],
            "competitive_turn_taking": ["Chess", "Poker", "Esports"]
        }
        
        sport = random.choice(sport_choices.get(sport_domain, ["Basketball"]))
        
        return cls.create(
            name=f"{sport} Professional League",
            sport=sport,
            region=fake.country(),
            tier=random.choice(["2.1", "3.1", "4.1"]),
            partnership_score=random.uniform(50, 85)
        )

name: Test Suite
 
on:
  push:
    branches: [ main, develop ]
  pull_request:
    branches: [ main, develop ]
 
env:
  PYTHON_VERSION: "3.11"
  NODE_VERSION: "20"
 
jobs:
  backend-tests:
    runs-on: ubuntu-latest
 
    services:
      postgres:
        image: postgres:15
        env:
          POSTGRES_PASSWORD: postgres
          POSTGRES_DB: test_db
        ports:
          - 5432:5432
 
      redis:
        image: redis:7
        ports:
          - 6379:6379
 
    steps:
      - uses: actions/checkout@v4
 
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ env.PYTHON_VERSION }}
 
      - name: Install dependencies
        run: |
          pip install -r apps/backend/requirements.txt
          pip install pytest pytest-asyncio pytest-cov
 
      - name: Run unit tests
        run: |
          cd apps/backend
          pytest tests/unit -v --cov=. --cov-report=xml
 
      - name: Run integration tests
        env:
          DATABASE_URL: postgresql://postgres:postgres@localhost:5432/test_db
          REDIS_URL: redis://localhost:6379
        run: |
          cd apps/backend
          pytest tests/integration -v
 
      - name: Upload coverage
        uses: codecov/codecov-action@v3
        with:
          file: ./apps/backend/coverage.xml
          flags: backend
 
  frontend-tests:
    runs-on: ubuntu-latest
 
    steps:
      - uses: actions/checkout@v4
 
      - name: Set up Node.js
        uses: actions/setup-node@v4
        with:
          node-version: ${{ env.NODE_VERSION }}
 
      - name: Install dependencies
        run: |
          cd clients/frontend
          npm ci
 
      - name: Run unit tests
        run: |
          cd clients/frontend
          npm run test:unit
 
      - name: Run component tests
        run: |
          cd clients/frontend
          npm run test:components
 
      - name: Build application
        run: |
          cd clients/frontend
          npm run build
 
  e2e-tests:
    runs-on: ubuntu-latest
    needs: [backend-tests, frontend-tests]
 
    steps:
      - uses: actions/checkout@v4
 
      - name: Set up Node.js
        uses: actions/setup-node@v4
        with:
          node-version: ${{ env.NODE_VERSION }}
 
      - name: Install Playwright
        run: |
          cd clients/frontend
          npm ci
          npx playwright install --with-deps
 
      - name: Run E2E tests
        run: |
          cd clients/frontend
          npm run test:e2e
 
      - name: Upload Playwright report
        if: always()
        uses: actions/upload-artifact@v3
        with:
          name: playwright-report
          path: clients/frontend/playwright-report/
 
  security-scan:
    runs-on: ubuntu-latest
 
    steps:
      - uses: actions/checkout@v4
 
      - name: Run Bandit (Python)
        run: |
          pip install bandit
          bandit -r apps/backend -f json -o bandit-report.json
 
      - name: Run npm audit (Frontend)
        run: |
          cd clients/frontend
          npm audit --audit-level=moderate
 
      - name: Run Snyk scan
        uses: snyk/actions/python@master
        env:
          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
        with:
          args: --severity-threshold=high
 
  quality-gate:
    runs-on: ubuntu-latest
    needs: [backend-tests, frontend-tests, e2e-tests, security-scan]
 
    steps:
      - name: Check quality metrics
        run: |
          echo "All tests passed!"
          echo "Quality gate: PASSED ✅"

# tests/orchestrator/test_orchestrator.py
import asyncio
from typing import List, Dict, Any
from concurrent.futures import ThreadPoolExecutor
import pytest
 
from tests.fixtures.test_fixtures import TestSuite, TestResult
 
class TestOrchestrator:
    """Parallel test execution orchestrator"""
 
    def __init__(self, max_workers: int = None):
        self.max_workers = max_workers or os.cpu_count()
        self.executor = ThreadPoolExecutor(max_workers=self.max_workers)
 
    async def run_parallel_tests(self, test_suites: List[TestSuite]) -> AggregatedTestResult:
        """Run test suites in parallel"""
 
        all_results = []
        failed_suites = []
 
        # Run each suite in parallel
        tasks = [
            self.executor.submit(self._run_test_suite, suite)
            for suite in test_suites
        ]
 
        # Collect results
        for future in asyncio.as_completed(tasks):
            try:
                result = future.result()
                all_results.extend(result.results)
                if result.failed_tests:
                    failed_suites.append(result.suite_name)
            except Exception as e:
                # Log suite failure
                all_results.append(TestResult(
                    test_id="suite_error",
                    suite_name="unknown",
                    status="error",
                    error=str(e),
                    duration=0
                ))
 
        # Retry flaky tests
        flaky_results = await self._retry_flaky_tests(all_results)
        all_results.extend(flaky_results)
 
        # Aggregate results
        aggregated = self._aggregate_results(all_results)
 
        return AggregatedTestResult(
            total_tests=len(all_results),
            passed_tests=len([r for r in all_results if r.status == "passed"]),
            failed_tests=len([r for r in all_results if r.status == "failed"]),
            error_tests=len([r for r in all_results if r.status == "error"]),
            duration=sum(r.duration for r in all_results),
            flaky_retries=len(flaky_results),
            suite_results=all_results
        )
 
    def _run_test_suite(self, suite: TestSuite) -> TestSuiteResult:
        """Run individual test suite"""
        results = []
        
        for test in suite.tests:
            start_time = time.time()
            try:
                outcome = test.run()
                duration = time.time() - start_time
                results.append(TestResult(
                    test_id=test.id,
                    suite_name=suite.name,
                    status="passed" if outcome else "failed",
                    duration=duration
                ))
            except Exception as e:
                duration = time.time() - start_time
                results.append(TestResult(
                    test_id=test.id,
                    suite_name=suite.name,
                    status="error",
                    error=str(e),
                    duration=duration
                ))
 
        return TestSuiteResult(
            suite_name=suite.name,
            results=results,
            failed_tests=[r for r in results if r.status != "passed"]
        )
 
    async def _retry_flaky_tests(self, results: List[TestResult]) -> List[TestResult]:
        """Retry flaky tests up to 3 times"""
        flaky_tests = [r for r in results if r.status == "failed" and "flaky" in r.tags]
        retry_results = []
 
        for test_result in flaky_tests:
            for attempt in range(3):
                try:
                    # Rerun test
                    outcome = test_result.test.rerun()
                    duration = 0  # Measure actual duration
                    retry_results.append(TestResult(
                        test_id=test_result.test_id,
                        suite_name=test_result.suite_name,
                        status="passed" if outcome else "failed",
                        duration=duration,
                        retry_attempt=attempt + 1
                    ))
                    break
                except Exception as e:
                    duration = 0
                    retry_results.append(TestResult(
                        test_id=test_result.test_id,
                        suite_name=test_result.suite_name,
                        status="error",
                        error=f"Retry {attempt + 1} failed: {str(e)}",
                        duration=duration,
                        retry_attempt=attempt + 1
                    ))
 
        return retry_results
 
    def _aggregate_results(self, results: List[TestResult]) -> AggregatedTestResult:
        """Aggregate test results"""
        passed = len([r for r in results if r.status == "passed"])
        failed = len([r for r in results if r.status == "failed"])
        errors = len([r for r in results if r.status == "error"])
 
        return AggregatedTestResult(
            total= len(results),
            passed=passed,
            failed=failed,
            errors=errors,
            pass_rate=passed / len(results) if results else 0,
            results=results
        )

// clients/frontend/e2e/visual-regression.spec.ts
import { test, expect } from '@playwright/test';
import { defineConfig, devices } from '@playwright/test';
 
test.describe('Visual Regression Tests', () => {
  test.describe('League Dashboard', () => {
    test('Desktop view matches baseline', async ({ page }) => {
      await page.goto('/dashboard');
      
      // Wait for dynamic content to load
      await page.waitForLoadState('networkidle');
      
      // Capture screenshot of key sections
      await expect(page).toHaveScreenshot('dashboard-desktop.png', {
        fullPage: true,
        threshold: 0.1,  // 10% tolerance for minor changes
        mask: [
          // Mask dynamic elements
          page.locator('[data-testid="timestamp"]'),
          page.locator('[data-testid="user-menu"]')
        ]
      });
    });
 
    test('Mobile view matches baseline', async ({ page }) => {
      // Set mobile viewport
      await page.setViewportSize({ width: 375, height: 812 });
      
      await page.goto('/dashboard');
      await page.waitForLoadState('networkidle');
      
      await expect(page).toHaveScreenshot('dashboard-mobile.png', {
        fullPage: true,
        threshold: 0.1
      });
    });
 
    test('Dark mode matches baseline', async ({ page }) => {
      // Enable dark mode
      await page.evaluate(() => {
        localStorage.setItem('theme', 'dark');
        document.documentElement.classList.add('dark');
      });
      
      await page.goto('/dashboard');
      await page.waitForLoadState('networkidle');
      
      await expect(page).toHaveScreenshot('dashboard-dark.png', {
        fullPage: true,
        threshold: 0.1
      });
    });
  });
 
  test.describe('Compliance Dashboard', () => {
    test('Compliance cards render correctly', async ({ page }) => {
      await page.goto('/compliance');
      
      // Test different states
      const complianceStates = [
        { score: 100, status: 'complete' },
        { score: 75, status: 'in-progress' },
        { score: 25, status: 'needs-attention' }
      ];
 
      for (const state of complianceStates) {
        // Mock compliance data
        await page.evaluate((data) => {
          window.complianceData = data;
        }, state);
        
        // Reload to trigger re-render
        await page.reload();
        await page.waitForLoadState('networkidle');
        
        await expect(page.locator(`[data-score="${state.score}"]`)).toHaveScreenshot(
          `compliance-card-${state.status}.png`,
          { threshold: 0.05 }
        );
      }
    });
  });
});
 
// playwright.config.ts
import { defineConfig, devices } from '@playwright/test';
 
export default defineConfig({
  testDir: './e2e',
  fullyParallel: true,
  forbidOnly: !!process.env.CI,
  retries: process.env.CI ? 2 : 0,
  workers: process.env.CI ? 1 : undefined,
  reporter: [['html'], ['json', { outputFile: 'test-results.json' }]],
  use: {
    baseURL: 'http://localhost:3000',
    trace: 'on-first-retry',
    screenshot: 'only-on-failure',
    video: 'retain-on-failure',
  },
  projects: [
    {
      name: 'chromium',
      use: { ...devices['Desktop Chrome'] },
    },
    {
      name: 'firefox',
      use: { ...devices['Desktop Firefox'] },
    },
    {
      name: 'webkit',
      use: { ...devices['Desktop Safari'] },
    },
    {
      name: 'Mobile Chrome',
      use: { ...devices['Pixel 5'] },
    },
  ],
  webServer: {
    command: 'npm run start',
    url: 'http://localhost:3000',
    reuseExistingServer: !process.env.CI,
  },
});