feat: Implement comprehensive Shadow DOM support (#50)

.github/SECURITY.md

@@ -0,0 +1,116 @@
+# Security Policy
+
+## Supported Versions
+
+We actively support the following versions of PyDoll:
+
+| Version | Supported          |
+| ------- | ------------------ |
+| 2.0.x   | :white_check_mark: |
+| 1.x.x   | :x:                |
+
+## Reporting a Vulnerability
+
+We take security vulnerabilities seriously. If you discover a security vulnerability, please report it to us privately.
+
+### Where to Report
+
+Please report security vulnerabilities by:
+
+1. **Email**: Send details to [[email protected]] (replace with your actual security email)
+2. **GitHub Security Advisories**: Use the "Security" tab in this repository
+3. **Private Disclosure**: Contact the maintainers directly through GitHub
+
+### What to Include
+
+When reporting a vulnerability, please include:
+
+- A clear description of the vulnerability
+- Steps to reproduce the issue
+- Potential impact assessment
+- Suggested fix (if available)
+- Your contact information for follow-up
+
+### Response Timeline
+
+- **Acknowledgment**: Within 48 hours
+- **Initial Assessment**: Within 1 week
+- **Fix Development**: Depends on severity (1-4 weeks)
+- **Disclosure**: After fix is deployed
+
+## Security Best Practices
+
+### For Contributors
+
+1. **Dependency Management**
+   - Keep dependencies up to date
+   - Use exact version pinning for security-critical dependencies
+   - Regularly audit dependencies for vulnerabilities
+
+2. **Code Security**
+   - Follow secure coding practices
+   - Validate all user inputs
+   - Use type hints and static analysis tools
+   - Implement proper error handling
+
+3. **Testing**
+   - Include security test cases
+   - Test for common web vulnerabilities
+   - Use automated security scanning tools
+
+### For Users
+
+1. **Installation**
+   - Always install from official sources (PyPI)
+   - Verify package signatures when available
+   - Use virtual environments
+
+2. **Usage**
+   - Keep PyDoll updated to the latest version
+   - Follow the principle of least privilege
+   - Validate all user inputs in your applications
+
+## Security Features
+
+### Browser Security
+
+- **Sandboxing**: PyDoll runs browsers in isolated environments
+- **Network Controls**: Configurable network restrictions
+- **File System Access**: Limited file system access controls
+
+### Connection Security
+
+- **TLS/SSL**: Secure connections to browser instances
+- **Authentication**: Proper authentication mechanisms
+- **Input Validation**: All protocol messages are validated
+
+## Known Security Considerations
+
+### Browser Security Context
+
+PyDoll controls browser instances which have inherent security implications:
+
+1. **Execution Context**: JavaScript code execution in controlled environments
+2. **Network Access**: Browsers can make network requests
+3. **File System**: Limited file system access through browser APIs
+
+### Mitigation Strategies
+
+1. **Isolated Environments**: Run in containers or virtual machines when possible
+2. **Network Policies**: Implement network restrictions
+3. **Resource Limits**: Set appropriate resource limits
+4. **Monitoring**: Monitor browser activities
+
+## Compliance
+
+This project follows:
+
+- **OWASP Guidelines**: Web application security best practices
+- **NIST Framework**: Cybersecurity framework guidelines
+- **Industry Standards**: Following established security standards
+
+## Updates
+
+This security policy is reviewed and updated regularly. Last updated: [Current Date]
+
+For questions about this security policy, please contact the maintainers. 

.github/dependabot.yml

@@ -0,0 +1,71 @@
+version: 2
+updates:
+  # Python dependencies
+  - package-ecosystem: "pip"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+      day: "monday"
+      time: "06:00"
+    open-pull-requests-limit: 10
+    reviewers:
+      - "autoscrape-labs"
+    assignees:
+      - "autoscrape-labs"
+    commit-message:
+      prefix: "deps"
+      prefix-development: "deps-dev"
+      include: "scope"
+    labels:
+      - "dependencies"
+      - "security"
+    # Group related updates
+    groups:
+      production-dependencies:
+        patterns:
+          - "websockets"
+          - "aiohttp"
+          - "aiofiles"
+          - "bs4"
+      development-dependencies:
+        patterns:
+          - "ruff"
+          - "pytest*"
+          - "mypy"
+          - "mkdocs*"
+          - "taskipy"
+    # Security updates
+    allow:
+      - dependency-type: "direct"
+      - dependency-type: "indirect"
+    # Ignore certain updates if needed
+    ignore:
+      - dependency-name: "*"
+        update-types: ["version-update:semver-major"]
+
+  # GitHub Actions dependencies
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+      day: "monday"
+      time: "06:00"
+    open-pull-requests-limit: 5
+    reviewers:
+      - "autoscrape-labs"
+    assignees:
+      - "autoscrape-labs"
+    commit-message:
+      prefix: "ci"
+      include: "scope"
+    labels:
+      - "github-actions"
+      - "security"
+    # Group GitHub Actions updates
+    groups:
+      github-actions:
+        patterns:
+          - "actions/*"
+          - "codecov/*"
+          - "softprops/*"
+          - "peaceiris/*" 

.github/workflows/codeql.yml

@@ -0,0 +1,70 @@
+name: "CodeQL Security Scan"
+
+on:
+  push:
+    branches: [ "main", "master", "develop" ]
+  pull_request:
+    branches: [ "main", "master", "develop" ]
+  schedule:
+    - cron: '30 2 * * 1'  # Weekly on Monday at 2:30 AM UTC
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    timeout-minutes: 360
+    permissions:
+      # Required for all workflows
+      security-events: write
+      # Required to fetch internal or private CodeQL packs
+      packages: read
+      # Required for workflows in private repositories
+      actions: read
+      contents: read
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'python' ]
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    # Initializes the CodeQL tools for scanning
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v3
+      with:
+        languages: ${{ matrix.language }}
+        queries: +security-and-quality
+        # Override default language queries
+        config: |
+          paths-ignore:
+            - "tests/"
+            - "docs/"
+            - "examples/"
+          queries:
+            - uses: security-and-quality
+            - uses: security-experimental
+
+    # Set up Python
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.10'
+
+    # Install dependencies
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install poetry
+        poetry install
+
+    # Perform the CodeQL Analysis
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v3
+      with:
+        category: "/language:${{matrix.language}}"
+        upload: true
+        # Fail on high severity issues
+        # fail-on: error 

.github/workflows/deploy-docs.yml

@@ -1,36 +1,41 @@
-name: Deploy MkDocs to GitHub Pages
+name: Deploy Documentation
 
 on:
   push:
     branches:
       - main
+  workflow_dispatch:
 
 jobs:
   deploy:
     runs-on: ubuntu-latest
+    # Add security hardening
+    permissions:
+      contents: write
+      pages: write
+      id-token: write
 
     steps:
-    - name: Code Checkout
-      uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
 
-    - name: Setup Python
-      uses: actions/setup-python@v4
-      with:
-        python-version: '3.x'
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
 
-    - name: Install Dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install mkdocs
-        pip install mkdocs-material
-        pip install pymdown-extensions
-        pip install mkdocstrings[python]
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install poetry
+          poetry install
 
-    - name: Build the documentation
-      run: mkdocs build
+      - name: Build documentation
+        run: poetry run mkdocs build
 
-    - name: Deploy to GitHub Pages
-      uses: peaceiris/actions-gh-pages@v3
-      with:
-        github_token: ${{ secrets.GITHUB_TOKEN }}
-        publish_dir: ./site
+      - name: Deploy to GitHub Pages
+        uses: peaceiris/actions-gh-pages@v3
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: ./site

.github/workflows/mypy.yml

@@ -10,28 +10,52 @@ on:
 
 jobs:
   build:
-
     runs-on: ubuntu-latest
-
+    # Add security hardening
+    permissions:
+      contents: read
     strategy:
       max-parallel: 4
       matrix:
-        python-version: ["3.11"]
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Install Dependencies
+      # Add caching for better performance
+      - name: Cache Poetry dependencies
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pypoetry
+          key: poetry-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('**/poetry.lock') }}
+          restore-keys: |
+            poetry-${{ runner.os }}-${{ matrix.python-version }}-
+
+      - name: Install Poetry
         run: |
           python -m pip install --upgrade pip
-          python -m pip install mypy
-          python -m pip install -e .
-          python -m mypy --install-types --non-interactive pydoll 
+          python -m pip install poetry
+
+      - name: Configure Poetry
+        run: |
+          poetry config virtualenvs.create true
+          poetry config virtualenvs.in-project true
+
+      - name: Install dependencies
+        run: |
+          poetry install
+          poetry run pip install mypy
+
+      - name: Verify installation
+        run: |
+          python --version
+          poetry run mypy --version
+          poetry --version
 
       - name: mypy
-        run: python -m mypy .
+        run: poetry run mypy . --ignore-missing-imports