feat(sync): RN-1545: Fix bugs

packages/api-client/src/constants.ts

@@ -16,9 +16,9 @@ const productionSubdomains = [
   'psss',
   'psss-api',
   'report-api',
+  'sync-api',
   'entity-api',
   'data-table-api',
-  'sync-api',
   'www',
   'api', // this must go last in the array, otherwise it will be detected before e.g. admin-api
 ];

packages/central-server/src/permissions/assertions.js

@@ -1,24 +1,24 @@
+import { TUPAIA_ADMIN_PANEL_PERMISSION_GROUP } from '@tupaia/constants';
 import { ensure } from '@tupaia/tsutils';
 import { PermissionsError } from '@tupaia/utils';
-import { TUPAIA_ADMIN_PANEL_PERMISSION_GROUP } from './constants';
 
 /* Re-export for backward compatibility. Prefer importing directly from @tupaia/access-policy. */
 export {
   allowNoPermissions,
+  assertAdminPanelAccess,
   assertAllPermissions,
   assertAnyPermissions,
+  assertBESAdminAccess,
+  assertPermissionGroupAccess,
+  assertPermissionGroupsAccess,
+  assertVizBuilderAccess,
   hasBESAdminAccess,
-  hasVizBuilderAccess,
   hasPermissionGroupAccess,
   hasPermissionGroupsAccess,
   hasSomePermissionGroupsAccess,
-  assertBESAdminAccess,
-  assertVizBuilderAccess,
   hasTupaiaAdminPanelAccess,
   hasTupaiaAdminPanelAccessToCountry,
-  assertAdminPanelAccess,
-  assertPermissionGroupAccess,
-  assertPermissionGroupsAccess,
+  hasVizBuilderAccess,
 } from '@tupaia/access-policy';
 
 /**

packages/database/src/core/modelClasses/Entity.js

@@ -337,6 +337,10 @@ export class EntityRecord extends DatabaseRecord {
 export class EntityModel extends MaterializedViewLogDatabaseModel {
   static syncDirection = SyncDirections.BIDIRECTIONAL;
 
+  get excludedFieldsFromSync() {
+    return ['point', 'bounds', 'region', 'parent_id'];
+  }
+
   get DatabaseRecordClass() {
     return EntityRecord;
   }
@@ -707,9 +711,4 @@ export class EntityModel extends MaterializedViewLogDatabaseModel {
       groupBy: ['entity.id'],
     };
   }
-
-  sanitizeForClient = data => {
-    const { point, bounds, region, parent_id, ...sanitizedData } = data;
-    return sanitizedData;
-  };
 }

packages/database/src/core/modelClasses/SurveyResponse/leaderboard.js

@@ -32,7 +32,7 @@ export function getLeaderboardQuery(projectId = '') {
   return `
     SELECT r.user_id, user_account.first_name, user_account.last_name, r.coconuts, r.pigs
     FROM (
-      SELECT user_id, COUNT(*)::INT as coconuts, FLOOR(COUNT(*) / 100)::INT AS pigs
+      SELECT user_id, COUNT(*)::INT AS coconuts, FLOOR(COUNT(*) / 100)::INT AS pigs
       FROM survey_response
       JOIN survey ON survey.id=survey_id
       ${projectId ? 'WHERE survey.project_id = ?' : ''}

packages/database/src/core/sync/buildSyncLookupSelect.js

@@ -4,6 +4,7 @@ export async function buildSyncLookupSelect(model, columns = {}) {
   const attributes = Object.keys(await model.fetchSchema());
   const { projectIds } = columns;
   const table = model.databaseRecord;
+  const excludedFields = [...(model.excludedFieldsFromSync || []), ...COLUMNS_EXCLUDED_FROM_SYNC];
 
   return `
     SELECT
@@ -12,9 +13,7 @@ export async function buildSyncLookupSelect(model, columns = {}) {
       COALESCE(:updatedAtSyncTick, ${table}.updated_at_sync_tick),
       sync_device_tick.device_id,
       json_build_object(
-        ${attributes
-          .filter(a => !COLUMNS_EXCLUDED_FROM_SYNC.includes(a))
-          .map(a => `'${a}', ${table}.${a}`)}
+        ${attributes.filter(a => !excludedFields.includes(a)).map(a => `'${a}', ${table}.${a}`)}
       ),
       ${projectIds || 'NULL'}
   `;

packages/datatrak-web/jest.setup.js

@@ -16,13 +16,77 @@ Object.defineProperty(window, 'matchMedia', {
   })),
 });
 
-// TODO: Set up database for testing later
+const mockModels = {
+  localSystemFact: {
+    get: jest.fn().mockResolvedValue('test-device-id'),
+    set: jest.fn().mockResolvedValue(undefined),
+    addProjectForSync: jest.fn(),
+  },
+  closeDatabaseConnections: jest.fn(),
+};
 
 jest.mock('@tupaia/database', () => ({
   migrate: jest.fn(),
   ModelRegistry: jest.fn().mockImplementation(() => ({})),
 }));
 
-jest.mock('./src/database/DatatrakDatabase', () => ({
-  DatatrakDatabase: jest.fn().mockImplementation(() => ({})),
+jest.mock('./src/database/createDatabase', () => ({
+  createDatabase: jest.fn().mockImplementation(() => ({
+    models: {
+      localSystemFact: {
+        get: jest.fn().mockImplementation(arg => {
+          if (arg === 'deviceId') {
+            return 'test-device-id';
+          }
+          return undefined;
+        }),
+        addProjectForSync: jest.fn(),
+      },
+    },
+  })),
 }));
+
+jest.mock('./src/api/CurrentUserContext', () => {
+  const actual = jest.requireActual('./src/api/CurrentUserContext');
+
+  return {
+    ...actual,
+    useCurrentUserContext: jest.fn(() => ({
+      ...actual.useCurrentUserContext(), // Get the actual return value
+      accessPolicy: {}, // Override just this property
+    })),
+  };
+});
+
+// TODO: Set up database for testing later
+jest.mock('./src/api/DatabaseContext', () => {
+  const React = require('react');
+
+  return {
+    DatabaseContext: React.createContext({
+      models: mockModels,
+    }),
+    DatabaseProvider: ({ children }) => children,
+    useDatabaseContext: () => ({
+      models: mockModels,
+    }),
+  };
+});
+
+jest.mock('./src/api/SyncContext', () => {
+  const React = require('react');
+
+  return {
+    SyncContext: React.createContext({
+      clientSyncManager: {
+        triggerSync: jest.fn(),
+      },
+    }),
+    SyncProvider: ({ children }) => children,
+    useSyncContext: () => ({
+      clientSyncManager: {
+        triggerSync: jest.fn(),
+      },
+    }),
+  };
+});

packages/datatrak-web/src/__tests__/__integration__/login.test.tsx

@@ -40,7 +40,7 @@ describe('Login', () => {
     renderPage('/login');
     expect(await screen.findByRole('heading', { level: 2 })).toHaveTextContent('Log in');
     await doLogin();
-    server.use(mockUserRequest({ email: '[email protected]' }));
+    server.use(mockUserRequest({ email: '[email protected]', accessPolicy: [] }));
 
     expect(await screen.findByRole('heading', { level: 1 })).toHaveTextContent(/Select project/i);
   });
@@ -49,7 +49,7 @@ describe('Login', () => {
     renderPage('/login');
     expect(await screen.findByRole('heading', { level: 2 })).toHaveTextContent('Log in');
 
-    server.use(mockUserRequest({ email: '[email protected]', projectId: 'foo' }));
+    server.use(mockUserRequest({ email: '[email protected]', projectId: 'foo', accessPolicy: [] }));
     await doLogin();
 
     await screen.findByText(/Select survey/i);
@@ -59,7 +59,7 @@ describe('Login', () => {
     renderPage('/survey');
     expect(await screen.findByRole('heading', { level: 2 })).toHaveTextContent('Log in');
 
-    server.use(mockUserRequest({ email: '[email protected]', projectId: 'foo' }));
+    server.use(mockUserRequest({ email: '[email protected]', projectId: 'foo', accessPolicy: [] }));
     await doLogin();
 
     expect(await screen.findByRole('heading', { level: 1 })).toHaveTextContent(/Select survey/i);

packages/datatrak-web/src/__tests__/__integration__/survey.test.tsx

@@ -9,7 +9,14 @@ import { handlers } from '../mocks/handlers';
 const server = setupServer(
   ...handlers,
   rest.get('*/v1/getUser', (_, res, ctx) => {
-    return res(ctx.status(200), ctx.json({ name: 'John Smith', email: '[email protected]' }));
+    return res(
+      ctx.status(200),
+      ctx.json({
+        name: 'John Smith',
+        email: '[email protected]',
+        id: '0'.repeat(24),
+      }),
+    );
   }),
   rest.get('*/v1/*', (_, res, ctx) => {
     return res(ctx.status(200), ctx.json([]));

packages/datatrak-web/src/__tests__/features/Survey/CopySurveyUrlButton.test.tsx

@@ -9,7 +9,10 @@ import { handlers } from '../../mocks/handlers';
 const server = setupServer(
   ...handlers,
   rest.get('*/v1/getUser', (_, res, ctx) => {
-    return res(ctx.status(200), ctx.json({ name: 'John Smith', email: '[email protected]' }));
+    return res(
+      ctx.status(200),
+      ctx.json({ name: 'John Smith', email: '[email protected]', id: '0'.repeat(24) }),
+    );
   }),
 );
 

packages/datatrak-web/src/api/DatabaseContext.tsx

@@ -15,12 +15,18 @@ export const DatabaseProvider = ({ children }: { children: Readonly<React.ReactN
   const [models, setModels] = useState<DatatrakWebModelRegistry | null>(null);
 
   useEffect(() => {
+    let modelsInstance: DatatrakWebModelRegistry | null = null;
     const init = async () => {
       const { models } = await createDatabase();
+      modelsInstance = models;
       setModels(models);
     };
 
     init();
+
+    return () => {
+      modelsInstance?.closeDatabaseConnections();
+    };
   }, []);
 
   if (!models) {

packages/datatrak-web/src/api/mutations/useEditUser.ts

@@ -38,12 +38,13 @@ export const useEditUser = (onSuccess?: () => void) => {
       await put('me', { data: updates });
     },
     {
-      onSuccess: async (_, variables) => {
+      onSuccess: (_, variables) => {
         queryClient.invalidateQueries(['getUser']);
         // If the user changes their project, we need to invalidate the entity descendants query so that recent entities are updated if they change back to the previous project without refreshing the page
         if (variables.projectId) {
           queryClient.invalidateQueries(['entityDescendants']);
           queryClient.invalidateQueries(['tasks']);
+
           models.localSystemFact.addProjectForSync(variables.projectId);
         }
         onSuccess?.();

packages/datatrak-web/src/api/queries/useTask.ts

@@ -3,9 +3,9 @@ import { getTask } from '../../database';
 import { DatatrakWebModelRegistry } from '../../types';
 import { get } from '../api';
 import { useIsOfflineFirst } from '../offlineFirst';
-import { ContextualQueryFunctionContext, useDatabaseQuery } from './useDatabaseQuery';
+import { useDatabaseQuery } from './useDatabaseQuery';
 
-export interface UseTaskLocalContext extends ContextualQueryFunctionContext {
+export interface UseTaskLocalContext {
   models: DatatrakWebModelRegistry;
   taskId?: string;
 }

packages/datatrak-web/src/routes/PrivateRoute.tsx

@@ -1,16 +1,27 @@
-import React, { ReactElement } from 'react';
+import React, { ReactElement, useEffect } from 'react';
 import { Navigate, Outlet, useLocation } from 'react-router-dom';
 
 import { isFeatureEnabled } from '@tupaia/utils';
 
 import { useCurrentUserContext } from '../api';
 import { ADMIN_ONLY_ROUTES, ROUTES } from '../constants';
 import { isWebApp } from '../utils';
+import { useDatabaseContext } from '../hooks/database';
 
 // Reusable wrapper to handle redirecting to login if user is not logged in and the route is private
 export const PrivateRoute = ({ children }: { children?: ReactElement }): ReactElement => {
   const { isLoggedIn, hasAdminPanelAccess, hideWelcomeScreen, ...user } = useCurrentUserContext();
   const { pathname, search } = useLocation();
+  const { models } = useDatabaseContext();
+
+  useEffect(() => {
+    const addProjectForSync = async () => {
+      await models.localSystemFact.addProjectForSync(user.projectId);
+    };
+    if (isLoggedIn && user.projectId) {
+      addProjectForSync();
+    }
+  }, [models, isLoggedIn, user.projectId]);
 
   if (!isLoggedIn) {
     return (

packages/datatrak-web/src/sync/ClientSyncManager.ts

@@ -435,7 +435,7 @@ export class ClientSyncManager {
       pullProgressCallback(records.length);
     };
 
-    const batchSize = 200;
+    const batchSize = 10000;
     await pullIncomingChanges(this.models, sessionId, batchSize, processStreamedDataFunction);
 
     this.setProgress(this.progressMaxByStage[SYNC_STAGES.PERSIST - 1], 'Saving changes...');

packages/server-utils/src/ScheduledTask.ts

@@ -61,6 +61,8 @@ export class ScheduledTask {
    */
   models: DatabaseInterface;
 
+  isRunning = false;
+
   constructor(models: DatabaseInterface, name: string, schedule: string) {
     if (!name) {
       throw new Error(`ScheduledTask has no name`);
@@ -84,7 +86,16 @@ export class ScheduledTask {
   async runTask() {
     this.start = Date.now();
 
+    if (this.isRunning) {
+      const durationMs = Date.now() - this.start;
+      winston.info(`ScheduledTask: ${this.name}: Not running (previous task still running)`, {
+        durationMs,
+      });
+      return false;
+    }
+
     try {
+      this.isRunning = true;
       await this.models.wrapInTransaction(async (transactingModels: DatabaseInterface) => {
         // Acquire a database advisory lock for the transaction
         // Ensures no other server instance can execute its change handler at the same time
@@ -101,6 +112,7 @@ export class ScheduledTask {
       return false;
     } finally {
       this.start = null;
+      this.isRunning = false;
     }
   }
 

packages/superset-api/package.json

@@ -18,6 +18,7 @@
     "test": "yarn package:test"
   },
   "dependencies": {
+    "@tupaia/utils": "workspace:*",
     "https-proxy-agent": "^5.0.1",
     "node-fetch": "^1.7.3",
     "winston": "^3.17.0"