feat(sync): RN-1545: Fix bugs

packages/api-client/src/constants.ts

@@ -16,9 +16,9 @@ const productionSubdomains = [
   'psss',
   'psss-api',
   'report-api',
+  'sync-api',
   'entity-api',
   'data-table-api',
-  'sync-api',
   'www',
   'api', // this must go last in the array, otherwise it will be detected before e.g. admin-api
 ];

packages/central-server/src/permissions/assertions.js

@@ -1,24 +1,24 @@
+import { TUPAIA_ADMIN_PANEL_PERMISSION_GROUP } from '@tupaia/constants';
 import { ensure } from '@tupaia/tsutils';
 import { PermissionsError } from '@tupaia/utils';
-import { TUPAIA_ADMIN_PANEL_PERMISSION_GROUP } from './constants';
 
 /* Re-export for backward compatibility. Prefer importing directly from @tupaia/access-policy. */
 export {
   allowNoPermissions,
+  assertAdminPanelAccess,
   assertAllPermissions,
   assertAnyPermissions,
+  assertBESAdminAccess,
+  assertPermissionGroupAccess,
+  assertPermissionGroupsAccess,
+  assertVizBuilderAccess,
   hasBESAdminAccess,
-  hasVizBuilderAccess,
   hasPermissionGroupAccess,
   hasPermissionGroupsAccess,
   hasSomePermissionGroupsAccess,
-  assertBESAdminAccess,
-  assertVizBuilderAccess,
   hasTupaiaAdminPanelAccess,
   hasTupaiaAdminPanelAccessToCountry,
-  assertAdminPanelAccess,
-  assertPermissionGroupAccess,
-  assertPermissionGroupsAccess,
+  hasVizBuilderAccess,
 } from '@tupaia/access-policy';
 
 /**

packages/database/src/core/modelClasses/Entity.js

@@ -336,6 +336,10 @@ export class EntityRecord extends DatabaseRecord {
 export class EntityModel extends MaterializedViewLogDatabaseModel {
   static syncDirection = SyncDirections.BIDIRECTIONAL;
 
+  get ExcludedFieldsFromSync() {
+    return ['point', 'bounds', 'region', 'parent_id'];
+  }
+
   get DatabaseRecordClass() {
     return EntityRecord;
   }
@@ -699,9 +703,4 @@ export class EntityModel extends MaterializedViewLogDatabaseModel {
       groupBy: ['entity.id'],
     };
   }
-
-  sanitizeForClient = data => {
-    const { point, bounds, region, parent_id, ...sanitizedData } = data;
-    return sanitizedData;
-  };
 }

packages/database/src/core/modelClasses/SurveyResponse/leaderboard.js

@@ -32,7 +32,7 @@ export function getLeaderboardQuery(projectId = '') {
   return `
     SELECT r.user_id, user_account.first_name, user_account.last_name, r.coconuts, r.pigs
     FROM (
-      SELECT user_id, COUNT(*)::INT as coconuts, FLOOR(COUNT(*) / 100)::INT AS pigs
+      SELECT user_id, COUNT(*)::INT AS coconuts, FLOOR(COUNT(*) / 100)::INT AS pigs
       FROM survey_response
       JOIN survey ON survey.id=survey_id
       ${projectId ? 'WHERE survey.project_id = ?' : ''}

packages/database/src/core/sync/buildSyncLookupSelect.js

@@ -2,8 +2,9 @@ import { COLUMNS_EXCLUDED_FROM_SYNC } from '@tupaia/constants';
 
 export async function buildSyncLookupSelect(model, columns = {}) {
   const attributes = Object.keys(await model.fetchSchema());
-  const { projectIds, userIds } = columns;
+  const { projectIds } = columns;
   const table = model.databaseRecord;
+  const excludedFields = [...(model.ExcludedFieldsFromSync || []), ...COLUMNS_EXCLUDED_FROM_SYNC];
 
   return `
     SELECT
@@ -12,9 +13,7 @@ export async function buildSyncLookupSelect(model, columns = {}) {
       COALESCE(:updatedAtSyncTick, ${table}.updated_at_sync_tick),
       sync_device_tick.device_id,
       json_build_object(
-        ${attributes
-          .filter(a => !COLUMNS_EXCLUDED_FROM_SYNC.includes(a))
-          .map(a => `'${a}', ${table}.${a}`)}
+        ${attributes.filter(a => !excludedFields.includes(a)).map(a => `'${a}', ${table}.${a}`)}
       ),
       ${projectIds || 'NULL'}
   `;

packages/datatrak-web/jest.setup.js

@@ -1,10 +1,92 @@
-// TODO: Set up database for testing later
+/**
+ * This is the Jest-sanctioned workaround
+ * @see https://jestjs.io/docs/manual-mocks#mocking-methods-which-are-not-implemented-in-jsdom
+ */
+Object.defineProperty(window, 'matchMedia', {
+  writable: true,
+  value: jest.fn().mockImplementation(query => ({
+    matches: false,
+    media: query,
+    onchange: null,
+    addListener: jest.fn(), // deprecated
+    removeListener: jest.fn(), // deprecated
+    addEventListener: jest.fn(),
+    removeEventListener: jest.fn(),
+    dispatchEvent: jest.fn(),
+  })),
+});
+
+const mockModels = {
+  localSystemFact: {
+    get: jest.fn().mockResolvedValue('test-device-id'),
+    set: jest.fn().mockResolvedValue(undefined),
+    addProjectForSync: jest.fn(),
+  },
+  closeDatabaseConnections: jest.fn(),
+};
 
 jest.mock('@tupaia/database', () => ({
   migrate: jest.fn(),
   ModelRegistry: jest.fn().mockImplementation(() => ({})),
 }));
 
-jest.mock('./src/database/DatatrakDatabase', () => ({
-  DatatrakDatabase: jest.fn().mockImplementation(() => ({})),
+jest.mock('./src/database/createDatabase', () => ({
+  createDatabase: jest.fn().mockImplementation(() => ({
+    models: {
+      localSystemFact: {
+        get: jest.fn().mockImplementation(arg => {
+          if (arg === 'deviceId') {
+            return 'test-device-id';
+          }
+          return undefined;
+        }),
+        addProjectForSync: jest.fn(),
+      },
+    },
+  })),
 }));
+
+jest.mock('./src/api/CurrentUserContext', () => {
+  const actual = jest.requireActual('./src/api/CurrentUserContext');
+
+  return {
+    ...actual,
+    useCurrentUserContext: jest.fn(() => ({
+      ...actual.useCurrentUserContext(), // Get the actual return value
+      accessPolicy: {}, // Override just this property
+    })),
+  };
+});
+
+// TODO: Set up database for testing later
+jest.mock('./src/api/DatabaseContext', () => {
+  const React = require('react');
+
+  return {
+    DatabaseContext: React.createContext({
+      models: mockModels,
+    }),
+    DatabaseProvider: ({ children }) => children,
+    useDatabaseContext: () => ({
+      models: mockModels,
+    }),
+  };
+});
+
+jest.mock('./src/api/SyncContext', () => {
+  const React = require('react');
+
+  return {
+    SyncContext: React.createContext({
+      clientSyncManager: {
+        triggerSync: jest.fn(),
+      },
+    }),
+    SyncProvider: ({ children }) => children,
+    useSyncContext: () => ({
+      clientSyncManager: {
+        triggerSync: jest.fn(),
+      },
+    }),
+  };
+});

packages/datatrak-web/src/__tests__/__integration__/login.test.tsx

@@ -40,7 +40,7 @@ describe('Login', () => {
     renderPage('/login');
     expect(await screen.findByRole('heading', { level: 2 })).toHaveTextContent('Log in');
     await doLogin();
-    server.use(mockUserRequest({ email: '[email protected]' }));
+    server.use(mockUserRequest({ email: '[email protected]', accessPolicy: [] }));
 
     expect(await screen.findByRole('heading', { level: 1 })).toHaveTextContent(/Select project/i);
   });
@@ -49,7 +49,7 @@ describe('Login', () => {
     renderPage('/login');
     expect(await screen.findByRole('heading', { level: 2 })).toHaveTextContent('Log in');
 
-    server.use(mockUserRequest({ email: '[email protected]', projectId: 'foo' }));
+    server.use(mockUserRequest({ email: '[email protected]', projectId: 'foo', accessPolicy: [] }));
     await doLogin();
 
     await screen.findByText(/Select survey/i);
@@ -59,7 +59,7 @@ describe('Login', () => {
     renderPage('/survey');
     expect(await screen.findByRole('heading', { level: 2 })).toHaveTextContent('Log in');
 
-    server.use(mockUserRequest({ email: '[email protected]', projectId: 'foo' }));
+    server.use(mockUserRequest({ email: '[email protected]', projectId: 'foo', accessPolicy: [] }));
     await doLogin();
 
     expect(await screen.findByRole('heading', { level: 1 })).toHaveTextContent(/Select survey/i);

packages/datatrak-web/src/__tests__/__integration__/survey.test.tsx

@@ -9,7 +9,14 @@ import { handlers } from '../mocks/handlers';
 const server = setupServer(
   ...handlers,
   rest.get('*/v1/getUser', (_, res, ctx) => {
-    return res(ctx.status(200), ctx.json({ name: 'John Smith', email: '[email protected]' }));
+    return res(
+      ctx.status(200),
+      ctx.json({
+        name: 'John Smith',
+        email: '[email protected]',
+        id: '0'.repeat(24),
+      }),
+    );
   }),
   rest.get('*/v1/*', (_, res, ctx) => {
     return res(ctx.status(200), ctx.json([]));

packages/datatrak-web/src/__tests__/features/Survey/CopySurveyUrlButton.test.tsx

@@ -9,7 +9,10 @@ import { handlers } from '../../mocks/handlers';
 const server = setupServer(
   ...handlers,
   rest.get('*/v1/getUser', (_, res, ctx) => {
-    return res(ctx.status(200), ctx.json({ name: 'John Smith', email: '[email protected]' }));
+    return res(
+      ctx.status(200),
+      ctx.json({ name: 'John Smith', email: '[email protected]', id: '0'.repeat(24) }),
+    );
   }),
 );
 

packages/datatrak-web/src/api/DatabaseContext.tsx

@@ -15,12 +15,18 @@ export const DatabaseProvider = ({ children }: { children: Readonly<React.ReactN
   const [models, setModels] = useState<DatatrakWebModelRegistry | null>(null);
 
   useEffect(() => {
+    let modelsInstance: DatatrakWebModelRegistry | null = null;
     const init = async () => {
       const { models } = await createDatabase();
+      modelsInstance = models;
       setModels(models);
     };
 
     init();
+
+    return () => {
+      modelsInstance?.closeDatabaseConnections();
+    };
   }, []);
 
   if (!models) {

packages/datatrak-web/src/api/mutations/useEditUser.ts

@@ -3,7 +3,6 @@ import { useMutation, useQueryClient } from '@tanstack/react-query';
 import { useDatabaseContext } from '../../hooks/database';
 import { UserAccountDetails } from '../../types';
 import { put } from '../api';
-import { useSyncContext } from '../SyncContext';
 
 /**
  * Converts a string from camel case to snake case.
@@ -22,7 +21,6 @@ function camelToSnakeCase(camelCaseString: string): string {
 export const useEditUser = (onSuccess?: () => void) => {
   const queryClient = useQueryClient();
   const { models } = useDatabaseContext();
-  const { refetchSyncedProjectIds } = useSyncContext();
 
   return useMutation<any, Error, UserAccountDetails, unknown>(
     async (userDetails: UserAccountDetails) => {
@@ -41,16 +39,14 @@ export const useEditUser = (onSuccess?: () => void) => {
     },
     {
       onSuccess: async (_, variables) => {
-        queryClient.invalidateQueries(['getUser']);
+        await queryClient.invalidateQueries(['getUser']);
         // If the user changes their project, we need to invalidate the entity descendants query so that recent entities are updated if they change back to the previous project without refreshing the page
         if (variables.projectId) {
-          queryClient.invalidateQueries(['entityDescendants']);
-          queryClient.invalidateQueries(['tasks']);
-
+          await queryClient.invalidateQueries(['entityDescendants']);
+          await queryClient.invalidateQueries(['tasks']);
+          
           (async () => {
             await models.localSystemFact.addProjectForSync(variables.projectId);
-            // Trigger immediate refresh of synced project IDs to enable immediate syncing
-            refetchSyncedProjectIds();
           })();
         }
         onSuccess?.();

packages/datatrak-web/src/api/mutations/useLogin.ts

@@ -17,7 +17,7 @@ export const useLogin = () => {
   const navigate = useNavigate();
   const from = useFromLocation();
   const { models } = useDatabaseContext();
-  const { refetchSyncedProjectIds } = useSyncContext();
+  const { clientSyncManager } = useSyncContext();
 
   return useMutation<any, Error, LoginCredentials, unknown>(
     ({ email, password }: LoginCredentials) => {
@@ -40,11 +40,10 @@ export const useLogin = () => {
 
         if (user.preferences?.projectId) {
           await models.localSystemFact.addProjectForSync(user.preferences.projectId);
-
-          // Trigger immediate refresh of synced project IDs to enable immediate syncing
-          refetchSyncedProjectIds();
         }
 
+        await clientSyncManager.triggerSync();
+
         if (from) {
           navigate(from, { state: null });
         } else {

packages/datatrak-web/src/api/queries/useTask.ts

@@ -4,9 +4,9 @@ import { getTask } from '../../database';
 import { DatatrakWebModelRegistry } from '../../types';
 import { get } from '../api';
 import { useIsOfflineFirst } from '../offlineFirst';
-import { ContextualQueryFunctionContext, useDatabaseQuery } from './useDatabaseQuery';
+import { useDatabaseQuery } from './useDatabaseQuery';
 
-export interface UseTaskLocalContext extends ContextualQueryFunctionContext {
+export interface UseTaskLocalContext {
   models: DatatrakWebModelRegistry;
   taskId?: string;
 }

packages/datatrak-web/src/hooks/database/index.ts

@@ -1,3 +1 @@
 export * from './useDatabaseContext';
-export * from './useDatabaseEffect';
-export * from './useProjectsInSync';