feat(statistics): calculate stats for any entity type using a view (#69)

closes #25

Author: sleidig

.github/copilot-instructions.md

@@ -200,7 +200,7 @@ When generating code, please adhere to the following specific instructions:
 - If similar changes are required in multiple places, only implement in one place and ask for review before implementing
   elsewhere
 - If a change is complex or large, first suggest and approach broken into smaller parts and ask for review after each
-  part
+  part. Ask for confirmation after each part before proceeding with changes in Agent mode.
 - If unsure about a specific implementation detail, ask for clarification before proceeding
 - Do not change any code or tests that are unrelated to the direct task
 - Check the "Problems" tab in VSCode for TypeScript errors and any other issues after making changes before running

src/couchdb/couchdb-admin.controller.spec.ts

@@ -14,14 +14,12 @@ describe('CouchdbAdminController', () => {
     {
       name: 'test1.example.com',
       users: 10,
-      childrenTotal: 50,
-      childrenActive: 45,
+      entities: { Child: { all: 50, active: 45 }, User: { all: 5, active: 3 } },
     },
     {
       name: 'test2.example.com',
       users: 5,
-      childrenTotal: 30,
-      childrenActive: 28,
+      entities: { Child: { all: 30, active: 28 } },
     },
   ];
 
@@ -90,7 +88,9 @@ describe('CouchdbAdminController', () => {
         'inline; filename="statistics.csv"',
       );
       expect(mockResponse.send).toHaveBeenCalledWith(
-        expect.stringContaining('name,users,childrenTotal,childrenActive'),
+        expect.stringContaining(
+          'name,users,Child_all,Child_active,User_all,User_active',
+        ),
       );
     });
 

src/couchdb/couchdb-admin.controller.ts

@@ -179,7 +179,24 @@ export class CouchdbAdminController {
     if (format === 'csv') {
       res.setHeader('Content-Type', 'text/csv');
       res.setHeader('Content-Disposition', 'inline; filename="statistics.csv"');
-      res.send(Papa.unparse(statisticsData));
+
+      // replace statistics.entities (`{ Child: { all: 30, active: 25 }, User: { all: 5, active: 3 } }`)
+      // and add flattened properties for each entity type and status to the statistics object
+      // (e.g. `Child_all: 30, Child_active: 25, User_all: 5, User_active: 3`)
+      const flattenedData = statisticsData.map((stat) => {
+        const flattenedEntities = {};
+        for (const [entityType, counts] of Object.entries(stat.entities)) {
+          flattenedEntities[`${entityType}_all`] = counts.all;
+          flattenedEntities[`${entityType}_active`] = counts.active;
+        }
+        return {
+          name: stat.name,
+          users: stat.users,
+          ...flattenedEntities,
+        };
+      });
+
+      res.send(Papa.unparse(flattenedData));
     } else {
       res.json(statisticsData);
     }

src/couchdb/couchdb.service.ts

@@ -59,7 +59,14 @@ export class Couchdb {
     this.baseUrl = `https://${this.url}/db`;
   }
 
-  get(path: string) {
+  get<R = any>(path: string, db?: string): Promise<R> {
+    if (!path.startsWith('/')) {
+      path = '/' + path;
+    }
+    if (db) {
+      path = `/${db}${path}`;
+    }
+
     const httpConfig = { auth: this.auth };
     return firstValueFrom(
       this.http.get(`${this.baseUrl}/couchdb${path}`, httpConfig).pipe(
@@ -104,7 +111,14 @@ export class Couchdb {
     );
   }
 
-  put(path: string, data, headers?: any): Promise<any> {
+  put(path: string, data, db?: string, headers?: any): Promise<any> {
+    if (!path.startsWith('/')) {
+      path = '/' + path;
+    }
+    if (db) {
+      path = `/${db}${path}`;
+    }
+
     return firstValueFrom(
       this.http
         .put(`${this.baseUrl}/couchdb${path}`, data, {
@@ -121,6 +135,10 @@ export class Couchdb {
   }
 
   post(path: string, data, headers?: any): Promise<any> {
+    if (!path.startsWith('/')) {
+      path = '/' + path;
+    }
+
     return firstValueFrom(
       this.http
         .post(`${this.baseUrl}/couchdb${path}`, data, {
@@ -135,4 +153,8 @@ export class Couchdb {
         ),
     );
   }
+
+  find(query, db = 'app'): Promise<any[]> {
+    return this.post(`/${db}/_find`, query);
+  }
 }

src/couchdb/statistics/statistics.service.spec.ts

@@ -1,21 +1,31 @@
 import { Test, TestingModule } from '@nestjs/testing';
 import { StatisticsService } from './statistics.service';
-import { CouchdbService } from '../couchdb.service';
+import { Couchdb, CouchdbService } from '../couchdb.service';
 import { KeycloakService } from '../../keycloak/keycloak.service';
 import {
   CredentialsService,
   SystemCredentials,
 } from '../../credentials/credentials.service';
+import { HttpService } from '@nestjs/axios';
 
 describe('StatisticsService', () => {
   let service: StatisticsService;
   let mockCouchdbService: jest.Mocked<CouchdbService>;
   let mockKeycloakService: jest.Mocked<KeycloakService>;
   let mockCredentialsService: jest.Mocked<CredentialsService>;
+  let mockCouchdbInstance: jest.Mocked<Couchdb>;
 
   beforeEach(async () => {
+    // Create a mock Couchdb instance
+    mockCouchdbInstance = {
+      url: 'org1.example.com',
+      get: jest.fn(),
+      put: jest.fn(),
+    } as any;
+
     const mockCouchdb = {
-      runForAllOrgs: jest.fn(),
+      getCouchdb: jest.fn().mockReturnValue(mockCouchdbInstance),
+      runForAllOrgs: jest.fn(), // We'll replace this with the real implementation
     };
 
     const mockKeycloak = {
@@ -40,44 +50,208 @@ describe('StatisticsService', () => {
     mockCouchdbService = module.get(CouchdbService);
     mockKeycloakService = module.get(KeycloakService);
     mockCredentialsService = module.get(CredentialsService);
+
+    // Create real CouchdbService instance to get the real runForAllOrgs implementation
+    // but bind it to our mocked service so it uses our mocked getCouchdb method
+    const realCouchdbService = new CouchdbService({} as HttpService);
+    mockCouchdbService.runForAllOrgs =
+      realCouchdbService.runForAllOrgs.bind(mockCouchdbService);
+
+    // Mock the createOrUpdateStatisticsView method to avoid the undefined error
+    jest
+      .spyOn(service as any, 'createOrUpdateStatisticsView')
+      .mockResolvedValue(undefined);
   });
 
   it('should be defined', () => {
     expect(service).toBeDefined();
   });
 
-  it('should get statistics from all organizations', async () => {
+  it('should get statistics from all organizations using CouchDB views', async () => {
     const mockToken = 'mock-token';
     const mockCredentials: SystemCredentials[] = [
       { url: 'org1.example.com', password: 'password1' },
     ];
-    const mockResults = {
-      'org1.example.com': {
-        name: 'org1.example.com',
-        users: 10,
-        childrenTotal: 50,
-        childrenActive: 45,
-      },
-    };
+    const mockUsers = Array(10)
+      .fill({})
+      .map((_, i) => ({ id: `user${i}` }));
+
+    // Mock CouchDB view responses
+    const mockStatsAll = [
+      { key: 'Child', value: 50 },
+      { key: 'User', value: 10 },
+      { key: 'School', value: 5 },
+    ];
+
+    const mockStatsActive = [
+      { key: 'Child', value: 45 },
+      { key: 'User', value: 8 },
+      { key: 'School', value: 5 },
+    ];
 
     mockKeycloakService.getKeycloakToken.mockResolvedValue(mockToken);
     mockCredentialsService.getCredentials.mockReturnValue(mockCredentials);
-    mockCouchdbService.runForAllOrgs.mockResolvedValue(mockResults);
+    mockKeycloakService.getUsersFromKeycloak.mockResolvedValue(
+      mockUsers as any,
+    );
+
+    // Mock the CouchDB get calls for the statistics views
+    mockCouchdbInstance.get
+      .mockResolvedValueOnce(mockStatsAll) // entities_all view
+      .mockResolvedValueOnce(mockStatsActive); // entities_active view
 
     const result = await service.getStatistics();
 
+    // Verify the sequence of operations
     expect(mockKeycloakService.getKeycloakToken).toHaveBeenCalled();
     expect(mockCredentialsService.getCredentials).toHaveBeenCalled();
-    expect(mockCouchdbService.runForAllOrgs).toHaveBeenCalledWith(
-      mockCredentials,
-      expect.any(Function),
+    expect(mockCouchdbService.getCouchdb).toHaveBeenCalledWith(
+      'org1.example.com',
+      'password1',
+    );
+    expect(mockKeycloakService.getUsersFromKeycloak).toHaveBeenCalledWith(
+      'org1',
+      mockToken,
+    );
+
+    // Verify view queries
+    expect(mockCouchdbInstance.get).toHaveBeenCalledWith(
+      '_design/statistics/_view/entities_all?group=true',
+      'app',
     );
+    expect(mockCouchdbInstance.get).toHaveBeenCalledWith(
+      '_design/statistics/_view/entities_active?group=true',
+      'app',
+    );
+
+    // Verify result structure
     expect(result).toEqual([
       {
         name: 'org1.example.com',
         users: 10,
-        childrenTotal: 50,
-        childrenActive: 45,
+        entities: {
+          Child: { all: 50, active: 45 },
+          User: { all: 10, active: 8 },
+          School: { all: 5, active: 5 },
+        },
+      },
+    ]);
+  });
+
+  it('should handle keycloak failure and fallback to empty users array', async () => {
+    const mockToken = 'mock-token';
+    const mockCredentials: SystemCredentials[] = [
+      { url: 'org1.example.com', password: 'password1' },
+    ];
+
+    const mockStatsAll = [
+      { key: 'Child', value: 30 },
+      { key: 'User', value: 5 },
+    ];
+
+    const mockStatsActive = [
+      { key: 'Child', value: 25 },
+      { key: 'User', value: 3 },
+    ];
+
+    mockKeycloakService.getKeycloakToken.mockResolvedValue(mockToken);
+    mockCredentialsService.getCredentials.mockReturnValue(mockCredentials);
+    mockKeycloakService.getUsersFromKeycloak.mockRejectedValue(
+      new Error('Keycloak error'),
+    );
+
+    mockCouchdbInstance.get
+      .mockResolvedValueOnce(mockStatsAll)
+      .mockResolvedValueOnce(mockStatsActive);
+
+    const result = await service.getStatistics();
+
+    expect(mockKeycloakService.getUsersFromKeycloak).toHaveBeenCalledWith(
+      'org1',
+      mockToken,
+    );
+
+    // When Keycloak fails, should use empty users array (length 0)
+    expect(result).toEqual([
+      {
+        name: 'org1.example.com',
+        users: 0,
+        entities: {
+          Child: { all: 30, active: 25 },
+          User: { all: 5, active: 3 },
+        },
+      },
+    ]);
+  });
+
+  it('should handle CouchDB view failures gracefully', async () => {
+    const mockToken = 'mock-token';
+    const mockCredentials: SystemCredentials[] = [
+      { url: 'org1.example.com', password: 'password1' },
+    ];
+    const mockUsers = Array(5)
+      .fill({})
+      .map((_, i) => ({ id: `user${i}` }));
+
+    mockKeycloakService.getKeycloakToken.mockResolvedValue(mockToken);
+    mockCredentialsService.getCredentials.mockReturnValue(mockCredentials);
+    mockKeycloakService.getUsersFromKeycloak.mockResolvedValue(
+      mockUsers as any,
+    );
+
+    // Mock CouchDB view failures
+    mockCouchdbInstance.get
+      .mockRejectedValueOnce(new Error('View error')) // entities_all view fails
+      .mockRejectedValueOnce(new Error('View error')); // entities_active view fails
+
+    const result = await service.getStatistics();
+
+    expect(result).toEqual([
+      {
+        name: 'org1.example.com',
+        users: 5,
+        entities: {}, // Empty entities when both views fail
+      },
+    ]);
+  });
+
+  it('should handle partial view data correctly', async () => {
+    const mockToken = 'mock-token';
+    const mockCredentials: SystemCredentials[] = [
+      { url: 'org1.example.com', password: 'password1' },
+    ];
+    const mockUsers = Array(3)
+      .fill({})
+      .map((_, i) => ({ id: `user${i}` }));
+
+    // Mock data where active has entities not in all (edge case)
+    const mockStatsAll = [{ key: 'Child', value: 20 }];
+
+    const mockStatsActive = [
+      { key: 'Child', value: 18 },
+      { key: 'Note', value: 5 }, // Note exists in active but not in all
+    ];
+
+    mockKeycloakService.getKeycloakToken.mockResolvedValue(mockToken);
+    mockCredentialsService.getCredentials.mockReturnValue(mockCredentials);
+    mockKeycloakService.getUsersFromKeycloak.mockResolvedValue(
+      mockUsers as any,
+    );
+
+    mockCouchdbInstance.get
+      .mockResolvedValueOnce(mockStatsAll)
+      .mockResolvedValueOnce(mockStatsActive);
+
+    const result = await service.getStatistics();
+
+    expect(result).toEqual([
+      {
+        name: 'org1.example.com',
+        users: 3,
+        entities: {
+          Child: { all: 20, active: 18 },
+          Note: { all: 0, active: 5 }, // Handles entity in active but not in all
+        },
       },
     ]);
   });