fix and correct gdrive

Signed-off-by: Derek Anderson <[email protected]>

Author: dmikey

experimental/docs/gdrive_testing_with_credentials.md

@@ -0,0 +1,268 @@
+# Testing Google Drive Backend with Real Credentials
+
+This guide explains how to set up and test the Google Drive backend with actual Google Cloud credentials.
+
+## Prerequisites
+
+1. A Google Cloud Platform (GCP) account
+2. Access to Google Drive
+3. Python environment with required packages installed
+
+## Setup Steps
+
+### 1. Create a Google Cloud Project
+
+1. Go to [Google Cloud Console](https://console.cloud.google.com/)
+2. Create a new project or select an existing one
+3. Enable the Google Drive API:
+   - Go to "APIs & Services" > "Library"
+   - Search for "Google Drive API"
+   - Click "Enable"
+4. Enable the Google Sheets API:
+   - Search for "Google Sheets API" 
+   - Click "Enable"
+
+### 2. Set Up Authentication
+
+Choose one of the following methods:
+
+#### Option A: Service Account (Recommended for automated testing)
+
+1. Go to "APIs & Services" > "Credentials"
+2. Click "Create Credentials" > "Service Account"
+3. Fill in the service account details
+4. Click "Create and Continue"
+5. Skip role assignment (click "Continue")
+6. Click "Done"
+7. Click on the created service account
+8. Go to "Keys" tab
+9. Click "Add Key" > "Create New Key"
+10. Choose "JSON" format
+11. Download the JSON file and save it securely (e.g., `service-account-key.json`)
+
+#### Option B: OAuth 2.0 (For interactive testing)
+
+1. Go to "APIs & Services" > "Credentials"
+2. Click "Create Credentials" > "OAuth 2.0 Client IDs"
+3. Choose "Desktop application"
+4. Download the JSON file and save it as `oauth-credentials.json`
+
+### 3. Create a Test Folder in Google Drive
+
+1. Go to [Google Drive](https://drive.google.com)
+2. Create a new folder for testing (e.g., "Ragas Test Data")
+3. Right-click the folder > "Share"
+4. If using Service Account: Share with the service account email (found in the JSON file)
+5. Copy the folder ID from the URL (e.g., `https://drive.google.com/drive/folders/FOLDER_ID_HERE`)
+
+### 4. Install Dependencies
+
+```bash
+cd /Users/derekanderson/Projects/explodinggradients/ragas/experimental
+pip install -r ../requirements/gdrive.txt
+```
+
+## Running Tests with Real Credentials
+
+### Method 1: Environment Variables
+
+Set up environment variables for your credentials:
+
+```bash
+# For Service Account
+export GDRIVE_SERVICE_ACCOUNT_PATH="/path/to/your/service-account-key.json"
+export GDRIVE_FOLDER_ID="your_folder_id_here"
+
+# For OAuth (optional)
+export GDRIVE_OAUTH_CREDENTIALS_PATH="/path/to/your/oauth-credentials.json"
+```
+
+### Method 2: Direct File Paths
+
+Create a test configuration file or modify the test directly with your paths.
+
+## Test Examples
+
+### Basic Integration Test
+
+Create a file `test_gdrive_integration.py`:
+
+```python
+import os
+import pytest
+from ragas_experimental.backends.gdrive_backend import GDriveBackend
+from ragas_experimental.project.core import Project
+from pydantic import BaseModel
+
+class TestData(BaseModel):
+    question: str
+    answer: str
+    score: float
+
+def test_gdrive_backend_real_credentials():
+    """Test Google Drive backend with real credentials."""
+    
+    # Get credentials from environment
+    service_account_path = os.getenv("GDRIVE_SERVICE_ACCOUNT_PATH")
+    folder_id = os.getenv("GDRIVE_FOLDER_ID")
+    
+    if not service_account_path or not folder_id:
+        pytest.skip("Real Google Drive credentials not provided")
+    
+    # Test backend creation
+    backend = GDriveBackend(
+        folder_id=folder_id,
+        project_id="test_project",
+        dataset_id="test_dataset",
+        dataset_name="Integration Test Dataset",
+        type="datasets",
+        service_account_path=service_account_path
+    )
+    
+    # Test data operations
+    test_data = [
+        TestData(question="What is AI?", answer="Artificial Intelligence", score=0.9),
+        TestData(question="What is ML?", answer="Machine Learning", score=0.8)
+    ]
+    
+    # Test save
+    backend.save(test_data)
+    print("✅ Data saved successfully")
+    
+    # Test load
+    loaded_data = backend.load(TestData)
+    assert len(loaded_data) == 2
+    assert loaded_data[0].question == "What is AI?"
+    print("✅ Data loaded successfully")
+    
+    # Test update
+    test_data[0].score = 0.95
+    backend.save(test_data)
+    updated_data = backend.load(TestData)
+    assert updated_data[0].score == 0.95
+    print("✅ Data updated successfully")
+    
+    print("🎉 All integration tests passed!")
+
+def test_project_with_gdrive():
+    """Test Project creation with Google Drive backend."""
+    
+    service_account_path = os.getenv("GDRIVE_SERVICE_ACCOUNT_PATH")
+    folder_id = os.getenv("GDRIVE_FOLDER_ID")
+    
+    if not service_account_path or not folder_id:
+        pytest.skip("Real Google Drive credentials not provided")
+    
+    project = Project.create(
+        name="gdrive_test_project",
+        backend="gdrive",
+        gdrive_folder_id=folder_id,
+        gdrive_service_account_path=service_account_path
+    )
+    
+    # Create a dataset
+    dataset = project.create_dataset(
+        name="test_dataset",
+        schema=TestData
+    )
+    
+    # Add some data
+    test_data = [
+        TestData(question="Test Q1", answer="Test A1", score=0.7),
+        TestData(question="Test Q2", answer="Test A2", score=0.8)
+    ]
+    
+    dataset.add(test_data)
+    
+    # Retrieve data
+    retrieved_data = dataset.to_list()
+    assert len(retrieved_data) == 2
+    
+    print("🎉 Project integration test passed!")
+
+if __name__ == "__main__":
+    test_gdrive_backend_real_credentials()
+    test_project_with_gdrive()
+```
+
+## Running the Tests
+
+### Option 1: With Environment Variables
+
+```bash
+# Set your credentials
+export GDRIVE_SERVICE_ACCOUNT_PATH="/path/to/service-account-key.json"
+export GDRIVE_FOLDER_ID="your_folder_id"
+
+# Run the integration test
+python test_gdrive_integration.py
+
+# Or with pytest
+pytest test_gdrive_integration.py -v
+```
+
+### Option 2: Direct Execution
+
+Modify the test file to include your credentials directly (not recommended for production):
+
+```python
+# In test_gdrive_integration.py, replace environment variables with actual values
+service_account_path = "/path/to/your/service-account-key.json"
+folder_id = "your_actual_folder_id"
+```
+
+## Troubleshooting
+
+### Common Issues
+
+1. **Permission Denied**: Make sure the service account has access to the Google Drive folder
+2. **API Not Enabled**: Ensure both Google Drive API and Google Sheets API are enabled
+3. **Invalid Credentials**: Check that the JSON file path is correct and the file is valid
+4. **Folder Not Found**: Verify the folder ID is correct and accessible
+
+### Debug Mode
+
+Add debug logging to see what's happening:
+
+```python
+import logging
+logging.basicConfig(level=logging.DEBUG)
+
+# Your test code here
+```
+
+### Manual Verification
+
+After running tests, check your Google Drive folder to see if:
+1. A spreadsheet was created with your dataset name
+2. Data was properly written to the spreadsheet
+3. Updates are reflected correctly
+
+## Security Best Practices
+
+1. **Never commit credentials**: Add credential files to `.gitignore`
+2. **Use environment variables**: Don't hardcode paths in your code
+3. **Limit permissions**: Only grant necessary permissions to service accounts
+4. **Rotate keys**: Regularly rotate service account keys
+5. **Use separate test projects**: Don't use production GCP projects for testing
+
+## CI/CD Integration
+
+For automated testing in CI/CD:
+
+1. Store credentials as encrypted secrets
+2. Use service accounts with minimal permissions
+3. Create dedicated test folders/projects
+4. Clean up test data after runs
+
+```yaml
+# Example GitHub Actions step
+- name: Test Google Drive Backend
+  env:
+    GDRIVE_SERVICE_ACCOUNT_KEY: ${{ secrets.GDRIVE_SERVICE_ACCOUNT_KEY }}
+    GDRIVE_FOLDER_ID: ${{ secrets.GDRIVE_FOLDER_ID }}
+  run: |
+    echo "$GDRIVE_SERVICE_ACCOUNT_KEY" > service-account.json
+    export GDRIVE_SERVICE_ACCOUNT_PATH="./service-account.json"
+    pytest test_gdrive_integration.py
+```

experimental/examples/gdrive_backend_example.py

@@ -38,37 +38,22 @@ def example_oauth_setup():
     """Example using OAuth authentication."""
 
     # Set up environment variables (or pass directly to Project.create)
-    os.environ["GDRIVE_FOLDER_ID"] = "your_google_drive_folder_id_here"
-    os.environ["GDRIVE_CREDENTIALS_PATH"] = "path/to/your/credentials.json"
+    # os.environ["GDRIVE_FOLDER_ID"] = "your_google_drive_folder_id_here"
+    # os.environ["GDRIVE_CREDENTIALS_PATH"] = "path/to/your/credentials.json"
 
     # Create project with Google Drive backend
     project = Project.create(
         name="my_ragas_project",
         description="A project using Google Drive for storage",
         backend="gdrive",
-        gdrive_folder_id="your_google_drive_folder_id_here",
-        gdrive_credentials_path="path/to/your/credentials.json",
+        gdrive_folder_id="1HLvvtKLnwGWKTely0YDlJ397XPTQ77Yg",
+        gdrive_credentials_path="/Users/derekanderson/Downloads/credentials.json",
         gdrive_token_path="token.json"  # Will be created automatically
     )
 
     return project
 
 
-def example_service_account_setup():
-    """Example using Service Account authentication."""
-    
-    # Create project with Google Drive backend using service account
-    project = Project.create(
-        name="my_ragas_project",
-        description="A project using Google Drive for storage",
-        backend="gdrive",
-        gdrive_folder_id="your_google_drive_folder_id_here",
-        gdrive_service_account_path="path/to/your/service_account.json"
-    )
-    
-    return project
-
-
 def example_usage():
     """Example of using the Google Drive backend."""
 

experimental/ragas_experimental/backends/base.py

@@ -0,0 +1,46 @@
+"""Base classes for dataset backends."""
+
+from abc import ABC, abstractmethod
+import typing as t
+
+
+class DatasetBackend(ABC):
+    """Abstract base class for dataset backends.
+
+    All dataset storage backends must implement these methods.
+    """
+
+    @abstractmethod
+    def initialize(self, dataset):
+        """Initialize the backend with dataset information"""
+        pass
+
+    @abstractmethod
+    def get_column_mapping(self, model):
+        """Get mapping between model fields and backend columns"""
+        pass
+
+    @abstractmethod
+    def load_entries(self, model_class):
+        """Load all entries from storage"""
+        pass
+
+    @abstractmethod
+    def append_entry(self, entry):
+        """Add a new entry to storage and return its ID"""
+        pass
+
+    @abstractmethod
+    def update_entry(self, entry):
+        """Update an existing entry in storage"""
+        pass
+
+    @abstractmethod
+    def delete_entry(self, entry_id):
+        """Delete an entry from storage"""
+        pass
+
+    @abstractmethod
+    def get_entry_by_field(self, field_name: str, field_value: t.Any, model_class):
+        """Get an entry by field value"""
+        pass