From 4cb810eab1a26f9fade701544a257baaf7a49780 Mon Sep 17 00:00:00 2001 From: chris-bes Date: Thu, 2 Oct 2025 22:43:29 +1000 Subject: [PATCH 01/26] fix bugs --- packages/api-client/src/constants.ts | 2 +- packages/datatrak-web/jest.setup.js | 68 ++++- .../__tests__/__integration__/login.test.tsx | 6 +- .../datatrak-web/src/api/DatabaseContext.tsx | 4 + .../src/api/mutations/useLogin.ts | 7 +- .../datatrak-web/src/api/queries/useTask.ts | 4 +- .../datatrak-web/src/hooks/database/index.ts | 2 - .../src/hooks/database/useDatabaseEffect.ts | 78 ----- .../src/hooks/database/useProjectsInSync.ts | 9 - .../datatrak-web/src/routes/PrivateRoute.tsx | 13 +- .../datatrak-web/src/views/Sync/SyncPage.tsx | 7 +- .../nginx-template/servers.template.conf | 6 + packages/server-utils/src/ScheduledTask.ts | 12 + packages/superset-api/package.json | 1 + .../src/sync/CentralSyncManager.ts | 63 +++- packages/sync/src/utils/getDependencyOrder.ts | 60 ++-- packages/sync/src/utils/index.ts | 1 + packages/sync/src/utils/saveChanges.ts | 56 +++- .../sync/src/utils/saveIncomingChanges.ts | 4 +- packages/tsutils/src/task/formatFilters.ts | 17 +- .../src/api/mutations/useEditUser.ts | 56 +++- packages/types/src/schemas/schemas.ts | 280 ++++++++++++------ packages/types/src/types/models.ts | 9 +- yarn.lock | 1 + 24 files changed, 491 insertions(+), 275 deletions(-) delete mode 100644 packages/datatrak-web/src/hooks/database/useDatabaseEffect.ts delete mode 100644 packages/datatrak-web/src/hooks/database/useProjectsInSync.ts diff --git a/packages/api-client/src/constants.ts b/packages/api-client/src/constants.ts index 26cd72ddc7..badcb7794a 100644 --- a/packages/api-client/src/constants.ts +++ b/packages/api-client/src/constants.ts @@ -16,9 +16,9 @@ const productionSubdomains = [ 'psss', 'psss-api', 'report-api', + 'sync-api', 'entity-api', 'data-table-api', - 'sync-api', 'www', 'api', // this must go last in the array, otherwise it will be detected before e.g. admin-api ]; diff --git a/packages/datatrak-web/jest.setup.js b/packages/datatrak-web/jest.setup.js index 4857469882..462aedf501 100644 --- a/packages/datatrak-web/jest.setup.js +++ b/packages/datatrak-web/jest.setup.js @@ -1,10 +1,64 @@ +/** + * This is the Jest-sanctioned workaround + * @see https://jestjs.io/docs/manual-mocks#mocking-methods-which-are-not-implemented-in-jsdom + */ +Object.defineProperty(window, 'matchMedia', { + writable: true, + value: jest.fn().mockImplementation(query => ({ + matches: false, + media: query, + onchange: null, + addListener: jest.fn(), // deprecated + removeListener: jest.fn(), // deprecated + addEventListener: jest.fn(), + removeEventListener: jest.fn(), + dispatchEvent: jest.fn(), + })), +}); + // TODO: Set up database for testing later +jest.mock('./src/api/DatabaseContext', () => { + const React = require('react'); + + return { + DatabaseContext: React.createContext({ + models: { + localSystemFact: { + get: jest.fn().mockResolvedValue('test-device-id'), + set: jest.fn().mockResolvedValue(undefined), + addProjectForSync: jest.fn(), + }, + closeDatabaseConnections: jest.fn(), + }, + }), + DatabaseProvider: ({ children }) => children, + useDatabaseContext: () => ({ + models: { + localSystemFact: { + get: jest.fn().mockResolvedValue('test-device-id'), + set: jest.fn().mockResolvedValue(undefined), + addProjectForSync: jest.fn(), + }, + closeDatabaseConnections: jest.fn(), + }, + }), + }; +}); -jest.mock('@tupaia/database', () => ({ - migrate: jest.fn(), - ModelRegistry: jest.fn().mockImplementation(() => ({})), -})); +jest.mock('./src/api/SyncContext', () => { + const React = require('react'); -jest.mock('./src/database/DatatrakDatabase', () => ({ - DatatrakDatabase: jest.fn().mockImplementation(() => ({})), -})); + return { + SyncContext: React.createContext({ + clientSyncManager: { + triggerSync: jest.fn(), + }, + }), + SyncProvider: ({ children }) => children, + useSyncContext: () => ({ + clientSyncManager: { + triggerSync: jest.fn(), + }, + }), + }; +}); \ No newline at end of file diff --git a/packages/datatrak-web/src/__tests__/__integration__/login.test.tsx b/packages/datatrak-web/src/__tests__/__integration__/login.test.tsx index fd20282bf9..de6f655894 100644 --- a/packages/datatrak-web/src/__tests__/__integration__/login.test.tsx +++ b/packages/datatrak-web/src/__tests__/__integration__/login.test.tsx @@ -40,7 +40,7 @@ describe('Login', () => { renderPage('/login'); expect(await screen.findByRole('heading', { level: 2 })).toHaveTextContent('Log in'); await doLogin(); - server.use(mockUserRequest({ email: 'john@gmail.com' })); + server.use(mockUserRequest({ email: 'john@gmail.com', acccessPolicy: [] })); expect(await screen.findByRole('heading', { level: 1 })).toHaveTextContent(/Select project/i); }); @@ -49,7 +49,7 @@ describe('Login', () => { renderPage('/login'); expect(await screen.findByRole('heading', { level: 2 })).toHaveTextContent('Log in'); - server.use(mockUserRequest({ email: 'john@gmail.com', projectId: 'foo' })); + server.use(mockUserRequest({ email: 'john@gmail.com', projectId: 'foo', acccessPolicy: [] })); await doLogin(); await screen.findByText(/Select survey/i); @@ -59,7 +59,7 @@ describe('Login', () => { renderPage('/survey'); expect(await screen.findByRole('heading', { level: 2 })).toHaveTextContent('Log in'); - server.use(mockUserRequest({ email: 'john@gmail.com', projectId: 'foo' })); + server.use(mockUserRequest({ email: 'john@gmail.com', projectId: 'foo', acccessPolicy: [] })); await doLogin(); expect(await screen.findByRole('heading', { level: 1 })).toHaveTextContent(/Select survey/i); diff --git a/packages/datatrak-web/src/api/DatabaseContext.tsx b/packages/datatrak-web/src/api/DatabaseContext.tsx index 0479cc5e83..68a1ad3d77 100644 --- a/packages/datatrak-web/src/api/DatabaseContext.tsx +++ b/packages/datatrak-web/src/api/DatabaseContext.tsx @@ -21,6 +21,10 @@ export const DatabaseProvider = ({ children }: { children: Readonly { + models?.closeDatabaseConnections(); + }; }, []); if (!models) { diff --git a/packages/datatrak-web/src/api/mutations/useLogin.ts b/packages/datatrak-web/src/api/mutations/useLogin.ts index 27bf4dea04..6cf2e8a568 100644 --- a/packages/datatrak-web/src/api/mutations/useLogin.ts +++ b/packages/datatrak-web/src/api/mutations/useLogin.ts @@ -17,7 +17,7 @@ export const useLogin = () => { const navigate = useNavigate(); const from = useFromLocation(); const { models } = useDatabaseContext(); - const { refetchSyncedProjectIds } = useSyncContext(); + const { clientSyncManager } = useSyncContext(); return useMutation( ({ email, password }: LoginCredentials) => { @@ -40,11 +40,10 @@ export const useLogin = () => { if (user.preferences?.projectId) { await models.localSystemFact.addProjectForSync(user.preferences.projectId); - - // Trigger immediate refresh of synced project IDs to enable immediate syncing - refetchSyncedProjectIds(); } + await clientSyncManager.triggerSync(); + if (from) { navigate(from, { state: null }); } else { diff --git a/packages/datatrak-web/src/api/queries/useTask.ts b/packages/datatrak-web/src/api/queries/useTask.ts index d5a7c9f3a9..b2d03860c4 100644 --- a/packages/datatrak-web/src/api/queries/useTask.ts +++ b/packages/datatrak-web/src/api/queries/useTask.ts @@ -4,9 +4,9 @@ import { getTask } from '../../database'; import { DatatrakWebModelRegistry } from '../../types'; import { get } from '../api'; import { useIsOfflineFirst } from '../offlineFirst'; -import { ContextualQueryFunctionContext, useDatabaseQuery } from './useDatabaseQuery'; +import { useDatabaseQuery } from './useDatabaseQuery'; -export interface UseTaskLocalContext extends ContextualQueryFunctionContext { +export interface UseTaskLocalContext { models: DatatrakWebModelRegistry; taskId?: string; } diff --git a/packages/datatrak-web/src/hooks/database/index.ts b/packages/datatrak-web/src/hooks/database/index.ts index 22ef913d79..ff87ec2aef 100644 --- a/packages/datatrak-web/src/hooks/database/index.ts +++ b/packages/datatrak-web/src/hooks/database/index.ts @@ -1,3 +1 @@ export * from './useDatabaseContext'; -export * from './useDatabaseEffect'; -export * from './useProjectsInSync'; diff --git a/packages/datatrak-web/src/hooks/database/useDatabaseEffect.ts b/packages/datatrak-web/src/hooks/database/useDatabaseEffect.ts deleted file mode 100644 index 5e48809ec5..0000000000 --- a/packages/datatrak-web/src/hooks/database/useDatabaseEffect.ts +++ /dev/null @@ -1,78 +0,0 @@ -import { useEffect, useState } from 'react'; - -import { AccessPolicy } from '@tupaia/access-policy'; - -import { DatatrakWebModelRegistry } from '../../types'; -import { useDatabaseContext } from './useDatabaseContext'; -import { CurrentUser, useCurrentUserContext } from '../../api'; - -export interface ResultObject { - data: T | undefined; - error: Error | undefined; - isLoading: boolean; - isSuccess: boolean; - onFetch: () => void; -} - -export interface DatabaseEffectOptions { - enabled: boolean; - onError?: (e: Error) => void; - placeholderData?: T; -} - -export const useCancelableEffect = ( - fetcher: () => Promise | T, - dependencies: React.DependencyList = [], - options: DatabaseEffectOptions, -): ResultObject => { - const { enabled = true, placeholderData, onError } = options; - const [data, setData] = useState(placeholderData as T); - const [error, setError] = useState(undefined); - const [isSuccess, setIsSuccess] = useState(false); - const [isLoading, setIsLoading] = useState(true); - - const onFetch = async (isCancel?: () => boolean) => { - if (!enabled) { - return; - } - - setIsLoading(true); - try { - const result = await fetcher(); - if (!isCancel?.()) { - setData(result); - setIsSuccess(true); - } - } catch (e: any) { - setError(e); - onError?.(e); - } finally { - setIsLoading(false); - } - }; - - useEffect(() => { - let canceled = false; - onFetch(() => canceled); - return (): void => { - canceled = true; - }; - }, dependencies); - - return { data, error, isLoading, isSuccess, onFetch }; -}; - -export const useDatabaseEffect = ( - call: ( - models: DatatrakWebModelRegistry, - accessPolicy?: AccessPolicy, - user?: CurrentUser, - ) => T | Promise, - dependencies: React.DependencyList, - options: DatabaseEffectOptions = { enabled: true, onError: (_e: Error) => {} }, -): ResultObject => { - const { models } = useDatabaseContext(); - const { accessPolicy, ...user } = useCurrentUserContext(); - - return useCancelableEffect(() => call(models, accessPolicy, user), dependencies, options); -}; diff --git a/packages/datatrak-web/src/hooks/database/useProjectsInSync.ts b/packages/datatrak-web/src/hooks/database/useProjectsInSync.ts deleted file mode 100644 index 1821b0ae68..0000000000 --- a/packages/datatrak-web/src/hooks/database/useProjectsInSync.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { FACT_PROJECTS_IN_SYNC } from '@tupaia/constants'; -import { useDatabaseEffect } from './useDatabaseEffect'; - -export const useProjectsInSync = () => - useDatabaseEffect(async models => { - const syncedProjectsFact = await models.localSystemFact.get(FACT_PROJECTS_IN_SYNC); - const syncedProjectIds = syncedProjectsFact ? JSON.parse(syncedProjectsFact) : []; - return syncedProjectIds; - }, []); diff --git a/packages/datatrak-web/src/routes/PrivateRoute.tsx b/packages/datatrak-web/src/routes/PrivateRoute.tsx index 081fb4aaa1..d884af28ff 100644 --- a/packages/datatrak-web/src/routes/PrivateRoute.tsx +++ b/packages/datatrak-web/src/routes/PrivateRoute.tsx @@ -1,4 +1,4 @@ -import React, { ReactElement } from 'react'; +import React, { ReactElement, useEffect } from 'react'; import { Navigate, Outlet, useLocation } from 'react-router-dom'; import { isFeatureEnabled } from '@tupaia/utils'; @@ -6,11 +6,22 @@ import { isFeatureEnabled } from '@tupaia/utils'; import { useCurrentUserContext } from '../api'; import { ADMIN_ONLY_ROUTES, ROUTES } from '../constants'; import { isWebApp } from '../utils'; +import { useDatabaseContext } from '../hooks/database'; // Reusable wrapper to handle redirecting to login if user is not logged in and the route is private export const PrivateRoute = ({ children }: { children?: ReactElement }): ReactElement => { const { isLoggedIn, hasAdminPanelAccess, hideWelcomeScreen, ...user } = useCurrentUserContext(); const { pathname, search } = useLocation(); + const { models } = useDatabaseContext(); + + useEffect(() => { + const addProjectForSync = async () => { + await models.localSystemFact.addProjectForSync(user.projectId); + }; + if (isLoggedIn && user.projectId) { + addProjectForSync(); + } + }, [models, isLoggedIn, user.projectId]); if (!isLoggedIn) { return ( diff --git a/packages/datatrak-web/src/views/Sync/SyncPage.tsx b/packages/datatrak-web/src/views/Sync/SyncPage.tsx index bba7612b1b..e50ddd91b3 100644 --- a/packages/datatrak-web/src/views/Sync/SyncPage.tsx +++ b/packages/datatrak-web/src/views/Sync/SyncPage.tsx @@ -10,7 +10,6 @@ import { SyncStatus } from './SyncStatus'; import { SYNC_EVENT_ACTIONS } from '../../types'; import { formatDistance } from 'date-fns'; import { useSyncContext } from '../../api/SyncContext'; -import { useProjectsInSync } from '../../hooks/database/useProjectsInSync'; const Wrapper = styled.div` block-size: 100dvb; @@ -131,11 +130,9 @@ export const SyncPage = () => { }; }, [syncManager]); - const { data: projectsInSync } = useProjectsInSync(); - const manualSync = useCallback(() => { - syncManager.triggerUrgentSync(projectsInSync); - }, [projectsInSync, syncManager]); + syncManager.triggerUrgentSync(); + }, [syncManager]); const syncFinishedSuccessfully = syncStarted && !isSyncing && !isQueuing && !errorMessage; diff --git a/packages/devops/configs/nginx-template/servers.template.conf b/packages/devops/configs/nginx-template/servers.template.conf index 0ef94a2b4b..b10a328108 100644 --- a/packages/devops/configs/nginx-template/servers.template.conf +++ b/packages/devops/configs/nginx-template/servers.template.conf @@ -424,6 +424,9 @@ server { include /etc/nginx/default.d/*.conf; include /etc/nginx/h5bp/basic.conf; + # Disable buffering for streaming responses + proxy_buffering off; + location / { proxy_pass http://localhost:8120; proxy_set_header Host $host; @@ -499,6 +502,9 @@ server { gzip_proxied no-cache no-store private expired auth; gzip_min_length 10000; + # Disable buffering for streaming responses + proxy_buffering off; + __HTTPS_CONFIG__ # Load configuration files for the default server block. diff --git a/packages/server-utils/src/ScheduledTask.ts b/packages/server-utils/src/ScheduledTask.ts index d3c4b6e7b0..f9189d3c02 100644 --- a/packages/server-utils/src/ScheduledTask.ts +++ b/packages/server-utils/src/ScheduledTask.ts @@ -61,6 +61,8 @@ export class ScheduledTask { */ models: DatabaseInterface; + isRunning = false; + constructor(models: DatabaseInterface, name: string, schedule: string) { if (!name) { throw new Error(`ScheduledTask has no name`); @@ -84,7 +86,16 @@ export class ScheduledTask { async runTask() { this.start = Date.now(); + if (this.isRunning) { + const durationMs = Date.now() - this.start; + winston.info(`ScheduledTask: ${this.name}: Not running (previous task still running)`, { + durationMs, + }); + return false; + } + try { + this.isRunning = true; await this.models.wrapInTransaction(async (transactingModels: DatabaseInterface) => { // Acquire a database advisory lock for the transaction // Ensures no other server instance can execute its change handler at the same time @@ -101,6 +112,7 @@ export class ScheduledTask { return false; } finally { this.start = null; + this.isRunning = false; } } diff --git a/packages/superset-api/package.json b/packages/superset-api/package.json index da270e3b4d..01479506af 100644 --- a/packages/superset-api/package.json +++ b/packages/superset-api/package.json @@ -18,6 +18,7 @@ "test": "yarn package:test" }, "dependencies": { + "@tupaia/utils": "workspace:*", "https-proxy-agent": "^5.0.1", "node-fetch": "^1.7.3", "winston": "^3.17.0" diff --git a/packages/sync-server/src/sync/CentralSyncManager.ts b/packages/sync-server/src/sync/CentralSyncManager.ts index c71f7c42ec..a31ef8f4d1 100644 --- a/packages/sync-server/src/sync/CentralSyncManager.ts +++ b/packages/sync-server/src/sync/CentralSyncManager.ts @@ -17,6 +17,7 @@ import { updateSnapshotRecords, SyncSnapshotAttributes, withDeferredSyncSafeguards, + findLastSuccessfulSyncedProjects, } from '@tupaia/sync'; import { objectIdToTimestamp } from '@tupaia/server-utils'; import { SyncTickFlags, FACT_CURRENT_SYNC_TICK, FACT_LOOKUP_UP_TO_TICK } from '@tupaia/constants'; @@ -255,7 +256,7 @@ export class CentralSyncManager { if (params.projectIds?.length === 0) { throw new Error('No project IDs provided'); } - + await this.connectToSession(sessionId); // first check if the snapshot is already being processed, to throw a sane error if (for some @@ -335,7 +336,7 @@ export class CentralSyncManager { unmarkSessionAsProcessing: () => Promise, accessPolicy: AccessPolicy, ): Promise { - const { since, projectIds, userId, deviceId } = snapshotParams; + const { since, projectIds, deviceId } = snapshotParams; let transactionTimeout; try { await this.connectToSession(sessionId); @@ -373,17 +374,59 @@ export class CentralSyncManager { }, snapshotTransactionTimeoutMs); } - // full changes - await snapshotOutgoingChanges( + performance.mark('start-findLastSuccessfulSyncedProjects'); + const lastSuccessfulSyncedProjectIds = await findLastSuccessfulSyncedProjects( transactingModels.database, - getModelsForPull(transactingModels.getModels()), - since, - sessionId, deviceId, - userId, - projectIds, - this.config, ); + performance.mark('end-findLastSuccessfulSyncedProjects'); + log.info( + 'findLastSuccessfulSyncedProjects', + performance.measure( + 'findLastSuccessfulSyncedProjects', + 'start-findLastSuccessfulSyncedProjects', + 'end-findLastSuccessfulSyncedProjects', + ), + ); + + const existingProjectIds = lastSuccessfulSyncedProjectIds.filter(projectId => + lastSuccessfulSyncedProjectIds.includes(projectId), + ); + const newProjectIds = projectIds.filter( + projectId => !lastSuccessfulSyncedProjectIds.includes(projectId), + ); + + // regular changes + if (existingProjectIds.length > 0) { + log.info('Snapshotting existing projectssss', { + existingProjectIds, + }); + await snapshotOutgoingChanges( + transactingModels.database, + getModelsForPull(transactingModels.getModels()), + since, + sessionId, + deviceId, + existingProjectIds, + this.config, + ); + } + + // full changes + if (newProjectIds.length > 0) { + log.info('Snapshotting new projectsss', { + newProjectIds, + }); + await snapshotOutgoingChanges( + transactingModels.database, + getModelsForPull(transactingModels.getModels()), + -1, + sessionId, + deviceId, + newProjectIds, + this.config, + ); + } await removeSnapshotDataByPermissions( sessionId, diff --git a/packages/sync/src/utils/getDependencyOrder.ts b/packages/sync/src/utils/getDependencyOrder.ts index c707e1a927..6377d22e47 100644 --- a/packages/sync/src/utils/getDependencyOrder.ts +++ b/packages/sync/src/utils/getDependencyOrder.ts @@ -1,6 +1,6 @@ import { compact, groupBy, mapValues } from 'lodash'; -import { BaseDatabase } from '@tupaia/database'; +import { BaseDatabase, DatabaseModel } from '@tupaia/database'; interface Dependency { table_name: string; @@ -11,36 +11,29 @@ export async function getDependencyOrder(database: BaseDatabase): Promise => { + const orderedDependencies = await getDependencyOrder(models[0].database); + const recordNames = new Set(models.map(r => r.databaseRecord)); + + return orderedDependencies + .filter(dep => recordNames.has(dep)) + .map(dep => models.find(r => r.databaseRecord === dep)) as DatabaseModel[]; +}; diff --git a/packages/sync/src/utils/index.ts b/packages/sync/src/utils/index.ts index cf844df65d..c03806e047 100644 --- a/packages/sync/src/utils/index.ts +++ b/packages/sync/src/utils/index.ts @@ -9,3 +9,4 @@ export * from './completeSyncSession'; export * from './sanitizeRecord'; export * from './startSnapshotWhenCapacityAvailable'; export * from './withDeferredSyncSafeguards'; +export * from './findLastSuccessfulSyncedProjects'; diff --git a/packages/sync/src/utils/saveChanges.ts b/packages/sync/src/utils/saveChanges.ts index 295a9ecdac..73c99cee05 100644 --- a/packages/sync/src/utils/saveChanges.ts +++ b/packages/sync/src/utils/saveChanges.ts @@ -8,8 +8,22 @@ export const saveCreates = async ( ) => { for (let i = 0; i < records.length; i += batchSize) { const batch = records.slice(i, i + batchSize); - await model.createMany(batch); - progressCallback?.(batch.length); + try { + await model.createMany(batch); + progressCallback?.(batch.length); + } catch (e: any) { + // try records individually, some may succeed and we want to capture the + // specific one with the error + await Promise.all( + batch.map(async row => { + try { + await model.create(row); + } catch (error) { + throw new Error(`Insert failed with '${e.message}', recordId: ${row.id}`); + } + }), + ); + } } }; @@ -23,8 +37,23 @@ export const saveUpdates = async ( for (let i = 0; i < recordsToSave.length; i += batchSize) { const batch = recordsToSave.slice(i, i + batchSize); - await Promise.all(batch.map(r => model.update({ id: r.id }, r))); - progressCallback?.(batch.length); + try { + await Promise.all(batch.map(r => model.updateById(r.id, r))); + // await Promise.all(batch.map(r => model.updateById(r.id, r))); + progressCallback?.(batch.length); + } catch (e) { + // try records individually, some may succeed and we want to capture the + // specific one with the error + await Promise.all( + batch.map(async row => { + try { + await model.updateById(row.id, row); + } catch (error: any) { + throw new Error(`Update failed with '${error.message}', recordId: ${row.id}`); + } + }), + ); + } } }; @@ -36,7 +65,22 @@ export const saveDeletes = async ( ) => { for (let i = 0; i < recordsForDelete.length; i += batchSize) { const batch = recordsForDelete.slice(i, i + batchSize); - await model.delete({ id: batch.map(r => r.id) }); - progressCallback?.(batch.length); + try { + await model.delete({ id: batch.map(r => r.id) }); + + progressCallback?.(batch.length); + } catch (e) { + // try records individually, some may succeed and we want to capture the + // specific one with the error + await Promise.all( + batch.map(async row => { + try { + await model.delete({ id: row.id }); + } catch (error: any) { + throw new Error(`Delete failed with '${error.message}', recordId: ${row.id}`); + } + }), + ); + } } }; diff --git a/packages/sync/src/utils/saveIncomingChanges.ts b/packages/sync/src/utils/saveIncomingChanges.ts index 7c382c6d46..24f01ebdb6 100644 --- a/packages/sync/src/utils/saveIncomingChanges.ts +++ b/packages/sync/src/utils/saveIncomingChanges.ts @@ -7,6 +7,7 @@ import { sleep } from '@tupaia/utils'; import { saveCreates, saveDeletes, saveUpdates } from './saveChanges'; import { ModelSanitizeArgs, RecordType, SyncSnapshotAttributes } from '../types'; import { findSyncSnapshotRecords } from './findSyncSnapshotRecords'; +import { sortModelsByDependencyOrder } from './getDependencyOrder'; // TODO: Move this to a config model RN-1668 const PERSISTED_CACHE_BATCH_SIZE = 10000; @@ -130,8 +131,9 @@ export const saveIncomingSnapshotChanges = async ( } assertIsWithinTransaction(models[0].database); + const sortedModels = await sortModelsByDependencyOrder(models); - for (const model of models) { + for (const model of sortedModels) { await saveChangesForModelInBatches( model, sessionId, diff --git a/packages/tsutils/src/task/formatFilters.ts b/packages/tsutils/src/task/formatFilters.ts index 90c20acbab..394492723f 100644 --- a/packages/tsutils/src/task/formatFilters.ts +++ b/packages/tsutils/src/task/formatFilters.ts @@ -1,4 +1,4 @@ -import { endOfDay, startOfDay } from 'date-fns'; +import { sub } from 'date-fns'; import { isEmpty } from '../typeGuards'; export interface FormattedFilters { @@ -28,9 +28,10 @@ export const formatFilters = (filters: Record[]) => { } if (id === FILTERS.DUE_DATE) { - const date = new Date(value); - const start = startOfDay(date); - const end = endOfDay(date); + // set the time to the end of the day to get the full range of the day, and apply milliseconds to ensure the range is inclusive + const end = new Date(value); + // subtract 23 hours, 59 minutes, 59 seconds to get the start of the day. This is because the filters always send the end of the day, and we need a range to handle the values being saved in the database as unix timestamps based on the user's timezone. + const start = sub(end, { hours: 23, minutes: 59, seconds: 59 }); formattedFilters[id] = { comparator: 'BETWEEN', comparisonValue: [start.getTime(), end.getTime()], @@ -38,13 +39,13 @@ export const formatFilters = (filters: Record[]) => { continue; } - if (id === FILTERS.REPEAT_SCHEDULE) { - formattedFilters['repeat_schedule->freq'] = value; + if (EQUALITY_FILTERS.includes(id)) { + formattedFilters[id] = value; continue; } - if (EQUALITY_FILTERS.includes(id)) { - formattedFilters[id] = value; + if (id === FILTERS.REPEAT_SCHEDULE) { + formattedFilters['repeat_schedule->freq'] = value; continue; } diff --git a/packages/tupaia-web/src/api/mutations/useEditUser.ts b/packages/tupaia-web/src/api/mutations/useEditUser.ts index 3300f8782d..15dfe479b8 100644 --- a/packages/tupaia-web/src/api/mutations/useEditUser.ts +++ b/packages/tupaia-web/src/api/mutations/useEditUser.ts @@ -1,25 +1,55 @@ import { useMutation, useQueryClient } from '@tanstack/react-query'; + +import { useDatabaseContext } from '../../hooks/database'; +import { UserAccountDetails } from '../../types'; import { put } from '../api'; -type UserAccountDetails = Record; +/** + * Converts a string from camel case to snake case. + * + * @remarks + * Ignores whitespace characters, including wordspaces and newlines. Does not handle fully- + * uppercase acronyms/initialisms. e.g. 'HTTPRequest' -> 'h_t_t_p_request'. + */ +function camelToSnakeCase(camelCaseString: string): string { + return camelCaseString + ?.split(/\.?(?=[A-Z])/) + .join('_') + .toLowerCase(); +} -export const useEditUser = (onSuccess?: (data: UserAccountDetails) => void) => { +export const useEditUser = (onSuccess?: () => void) => { const queryClient = useQueryClient(); + const { models } = useDatabaseContext(); return useMutation( - async ({ projectId }: Record) => { - const data = { - project_id: projectId, - }; - await put('me', { data }); - return { - projectId, - }; + async (userDetails: UserAccountDetails) => { + if (!userDetails) return; + + // `mobile_number` field in database is nullable; don't just store an empty string + if (!userDetails?.mobileNumber) { + userDetails.mobileNumber = null; + } + + const updates = Object.fromEntries( + Object.entries(userDetails).map(([key, value]) => [camelToSnakeCase(key), value]), + ); + + await put('me', { data: updates }); }, { - onSuccess: data => { - queryClient.invalidateQueries(['getUser']); - if (onSuccess) onSuccess(data); + onSuccess: async (_, variables) => { + await queryClient.invalidateQueries(['getUser']); + // If the user changes their project, we need to invalidate the entity descendants query so that recent entities are updated if they change back to the previous project without refreshing the page + if (variables.projectId) { + await queryClient.invalidateQueries(['entityDescendants']); + await queryClient.invalidateQueries(['tasks']); + + (async () => { + await models.localSystemFact.addProjectForSync(variables.projectId); + })(); + } + onSuccess?.(); }, }, ); diff --git a/packages/types/src/schemas/schemas.ts b/packages/types/src/schemas/schemas.ts index 129af06e1b..97d6970ea0 100644 --- a/packages/types/src/schemas/schemas.ts +++ b/packages/types/src/schemas/schemas.ts @@ -42040,13 +42040,17 @@ export const CountrySchema = { }, "name": { "type": "string" + }, + "updated_at_sync_tick": { + "type": "string" } }, "additionalProperties": false, "required": [ "code", "id", - "name" + "name", + "updated_at_sync_tick" ] } @@ -42058,6 +42062,9 @@ export const CountryCreateSchema = { }, "name": { "type": "string" + }, + "updated_at_sync_tick": { + "type": "string" } }, "additionalProperties": false, @@ -42075,6 +42082,9 @@ export const CountryUpdateSchema = { }, "name": { "type": "string" + }, + "updated_at_sync_tick": { + "type": "string" } }, "additionalProperties": false @@ -85172,8 +85182,6 @@ export const EntityTypeEnumSchema = { "incident_reported", "individual", "institute", - "kiuar_area", - "kiuar_facility", "larval_habitat", "larval_sample", "local_government", @@ -85190,7 +85198,6 @@ export const EntityTypeEnumSchema = { "pacmossi_spraying_site", "pacmossi_village", "pharmacy", - "policy", "postcode", "project", "repair_request", @@ -85199,9 +85206,6 @@ export const EntityTypeEnumSchema = { "sub_district", "sub_facility", "supermarket", - "tmf_district", - "tmf_facility", - "tmf_sub_district", "transfer", "trap", "vehicle", @@ -85303,18 +85307,67 @@ export const DebugLogUpdateSchema = { } export const ParamsSchema = { + "type": "object", + "additionalProperties": false +} + +export const ResBodySchema = { "type": "object", "properties": { - "projectCode": { + "status_code": { + "type": "number" + }, + "presentationConfig": { + "type": "object", + "additionalProperties": false + }, + "message": { "type": "string" } }, "additionalProperties": false, "required": [ - "projectCode" + "message", + "status_code" + ] +} + +export const ReqBodySchema = { + "type": "object", + "properties": { + "inputMessage": { + "type": "string" + }, + "dataStructure": { + "type": "object", + "additionalProperties": false + }, + "presentationOptions": { + "type": "object", + "additionalProperties": false + } + }, + "additionalProperties": false, + "required": [ + "dataStructure", + "inputMessage", + "presentationOptions" ] } +export const ReqQuerySchema = { + "type": "object", + "properties": { + "fields": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "additionalProperties": false +} + export const CountryAccessObjectSchema = { "type": "object", "properties": { @@ -85344,56 +85397,6 @@ export const CountryAccessObjectSchema = { ] } -export const ResBodySchema = { - "type": "array", - "items": { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "name": { - "type": "string" - }, - "code": { - "type": "string" - }, - "hasAccess": { - "type": "boolean" - }, - "hasPendingAccess": { - "type": "boolean" - } - }, - "additionalProperties": false, - "required": [ - "code", - "hasAccess", - "hasPendingAccess", - "id", - "name" - ] - } -} - -export const ReqBodySchema = { - "type": "object", - "additionalProperties": false -} - -export const ReqQuerySchema = { - "type": "object", - "properties": { - "projectId": { - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "projectId" - ] -} - export const IdSchema = { "type": "string" } @@ -86407,6 +86410,101 @@ export const TaskAssigneeSchema = { "additionalProperties": false } +export const RawTaskResultSchema = { + "additionalProperties": false, + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "updated_at_sync_tick": { + "type": "string" + }, + "entity_id": { + "type": "string" + }, + "survey_id": { + "type": "string" + }, + "assignee_id": { + "type": "string" + }, + "due_date": { + "type": "number" + }, + "initial_request_id": { + "type": "string" + }, + "overdue_email_sent": { + "type": "string", + "format": "date-time" + }, + "parent_task_id": { + "type": "string" + }, + "repeat_schedule": { + "type": "object", + "additionalProperties": false + }, + "status": { + "enum": [ + "cancelled", + "completed", + "to_do" + ], + "type": "string" + }, + "survey_response_id": { + "type": "string" + }, + "entity.name": { + "type": "string" + }, + "entity.code": { + "type": "string" + }, + "entity.country_code": { + "type": "string" + }, + "survey.code": { + "type": "string" + }, + "survey.name": { + "type": "string" + }, + "task_status": { + "enum": [ + "cancelled", + "completed", + "overdue", + "repeating", + "to_do" + ], + "type": "string" + }, + "task_due_date": { + "type": "string", + "format": "date-time" + }, + "assignee_name": { + "type": "string" + } + }, + "required": [ + "entity.code", + "entity.country_code", + "entity.name", + "entity_id", + "id", + "survey.code", + "survey.name", + "survey_id", + "task_due_date", + "task_status", + "updated_at_sync_tick" + ] +} + export const TaskResponseSchema = { "additionalProperties": false, "type": "object", @@ -86879,6 +86977,38 @@ export const UserResponseSchema = { ] } +export const QueueStatusSchema = { + "enum": [ + "activeSync", + "waitingInQueue" + ], + "type": "string" +} + +export const SyncPullReadyStatusSchema = { + "enum": [ + "pending", + "ready" + ], + "type": "string" +} + +export const SyncReadyStatusSchema = { + "enum": [ + "pending", + "ready" + ], + "type": "string" +} + +export const SyncPushStatusSchema = { + "enum": [ + "complete", + "pending" + ], + "type": "string" +} + export const CountriesResponseItemSchema = { "type": "object", "properties": { @@ -99232,35 +99362,3 @@ export const QuerySchema = { ] } -export const QueueStatusSchema = { - "enum": [ - "activeSync", - "waitingInQueue" - ], - "type": "string" -} - -export const SyncPullReadyStatusSchema = { - "enum": [ - "pending", - "ready" - ], - "type": "string" -} - -export const SyncReadyStatusSchema = { - "enum": [ - "pending", - "ready" - ], - "type": "string" -} - -export const SyncPushStatusSchema = { - "enum": [ - "complete", - "pending" - ], - "type": "string" -} - diff --git a/packages/types/src/types/models.ts b/packages/types/src/types/models.ts index e0448d41bd..3e0cf3c01a 100644 --- a/packages/types/src/types/models.ts +++ b/packages/types/src/types/models.ts @@ -255,14 +255,17 @@ export interface Country { 'code': string; 'id': string; 'name': string; + 'updated_at_sync_tick': string; } export interface CountryCreate { 'code': string; 'name': string; + 'updated_at_sync_tick'?: string; } export interface CountryUpdate { 'code'?: string; 'name'?: string; + 'updated_at_sync_tick'?: string; } export interface Dashboard { 'code': string; @@ -1979,12 +1982,6 @@ export enum EntityTypeEnum { 'consumable' = 'consumable', 'bes_asset' = 'bes_asset', 'bes_office' = 'bes_office', - 'tmf_district' = 'tmf_district', - 'tmf_sub_district' = 'tmf_sub_district', - 'tmf_facility' = 'tmf_facility', - 'policy' = 'policy', - 'kiuar_facility' = 'kiuar_facility', - 'kiuar_area' = 'kiuar_area', } export enum DataTableType { 'analytics' = 'analytics', diff --git a/yarn.lock b/yarn.lock index ead25c7a71..2b20d15950 100644 --- a/yarn.lock +++ b/yarn.lock @@ -13020,6 +13020,7 @@ __metadata: version: 0.0.0-use.local resolution: "@tupaia/superset-api@workspace:packages/superset-api" dependencies: + "@tupaia/utils": "workspace:*" "@types/jest": ^29.5.14 https-proxy-agent: ^5.0.1 jest: ^29.7.0 From 486c2940a61969787c8935974baf513cb9170512 Mon Sep 17 00:00:00 2001 From: chris-bes Date: Thu, 2 Oct 2025 23:16:41 +1000 Subject: [PATCH 02/26] fixed --- packages/sync-server/src/sync/CentralSyncManager.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/sync-server/src/sync/CentralSyncManager.ts b/packages/sync-server/src/sync/CentralSyncManager.ts index a31ef8f4d1..b00c2b4ca5 100644 --- a/packages/sync-server/src/sync/CentralSyncManager.ts +++ b/packages/sync-server/src/sync/CentralSyncManager.ts @@ -389,7 +389,7 @@ export class CentralSyncManager { ), ); - const existingProjectIds = lastSuccessfulSyncedProjectIds.filter(projectId => + const existingProjectIds = projectIds.filter(projectId => lastSuccessfulSyncedProjectIds.includes(projectId), ); const newProjectIds = projectIds.filter( From 7b161c040cbaf6b41b0939889241f2f48685e7eb Mon Sep 17 00:00:00 2001 From: chris-bes Date: Thu, 2 Oct 2025 23:18:17 +1000 Subject: [PATCH 03/26] fix --- .../datatrak-web/src/api/mutations/useEditUser.ts | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/packages/datatrak-web/src/api/mutations/useEditUser.ts b/packages/datatrak-web/src/api/mutations/useEditUser.ts index 3b3ab1544b..d41513803d 100644 --- a/packages/datatrak-web/src/api/mutations/useEditUser.ts +++ b/packages/datatrak-web/src/api/mutations/useEditUser.ts @@ -3,7 +3,6 @@ import { useMutation, useQueryClient } from '@tanstack/react-query'; import { useDatabaseContext } from '../../hooks/database'; import { UserAccountDetails } from '../../types'; import { put } from '../api'; -import { useSyncContext } from '../SyncContext'; /** * Converts a string from camel case to snake case. @@ -22,7 +21,6 @@ function camelToSnakeCase(camelCaseString: string): string { export const useEditUser = (onSuccess?: () => void) => { const queryClient = useQueryClient(); const { models } = useDatabaseContext(); - const { refetchSyncedProjectIds } = useSyncContext(); return useMutation( async (userDetails: UserAccountDetails) => { @@ -41,16 +39,14 @@ export const useEditUser = (onSuccess?: () => void) => { }, { onSuccess: async (_, variables) => { - queryClient.invalidateQueries(['getUser']); + await queryClient.invalidateQueries(['getUser']); // If the user changes their project, we need to invalidate the entity descendants query so that recent entities are updated if they change back to the previous project without refreshing the page if (variables.projectId) { - queryClient.invalidateQueries(['entityDescendants']); - queryClient.invalidateQueries(['tasks']); - + await queryClient.invalidateQueries(['entityDescendants']); + await queryClient.invalidateQueries(['tasks']); + (async () => { await models.localSystemFact.addProjectForSync(variables.projectId); - // Trigger immediate refresh of synced project IDs to enable immediate syncing - refetchSyncedProjectIds(); })(); } onSuccess?.(); From 2fa2bc4a8d7d7692b71e15a13173822e302062cf Mon Sep 17 00:00:00 2001 From: chris-bes Date: Thu, 2 Oct 2025 23:18:56 +1000 Subject: [PATCH 04/26] fix --- .../src/api/mutations/useEditUser.ts | 56 +++++-------------- 1 file changed, 13 insertions(+), 43 deletions(-) diff --git a/packages/tupaia-web/src/api/mutations/useEditUser.ts b/packages/tupaia-web/src/api/mutations/useEditUser.ts index 15dfe479b8..3300f8782d 100644 --- a/packages/tupaia-web/src/api/mutations/useEditUser.ts +++ b/packages/tupaia-web/src/api/mutations/useEditUser.ts @@ -1,55 +1,25 @@ import { useMutation, useQueryClient } from '@tanstack/react-query'; - -import { useDatabaseContext } from '../../hooks/database'; -import { UserAccountDetails } from '../../types'; import { put } from '../api'; -/** - * Converts a string from camel case to snake case. - * - * @remarks - * Ignores whitespace characters, including wordspaces and newlines. Does not handle fully- - * uppercase acronyms/initialisms. e.g. 'HTTPRequest' -> 'h_t_t_p_request'. - */ -function camelToSnakeCase(camelCaseString: string): string { - return camelCaseString - ?.split(/\.?(?=[A-Z])/) - .join('_') - .toLowerCase(); -} +type UserAccountDetails = Record; -export const useEditUser = (onSuccess?: () => void) => { +export const useEditUser = (onSuccess?: (data: UserAccountDetails) => void) => { const queryClient = useQueryClient(); - const { models } = useDatabaseContext(); return useMutation( - async (userDetails: UserAccountDetails) => { - if (!userDetails) return; - - // `mobile_number` field in database is nullable; don't just store an empty string - if (!userDetails?.mobileNumber) { - userDetails.mobileNumber = null; - } - - const updates = Object.fromEntries( - Object.entries(userDetails).map(([key, value]) => [camelToSnakeCase(key), value]), - ); - - await put('me', { data: updates }); + async ({ projectId }: Record) => { + const data = { + project_id: projectId, + }; + await put('me', { data }); + return { + projectId, + }; }, { - onSuccess: async (_, variables) => { - await queryClient.invalidateQueries(['getUser']); - // If the user changes their project, we need to invalidate the entity descendants query so that recent entities are updated if they change back to the previous project without refreshing the page - if (variables.projectId) { - await queryClient.invalidateQueries(['entityDescendants']); - await queryClient.invalidateQueries(['tasks']); - - (async () => { - await models.localSystemFact.addProjectForSync(variables.projectId); - })(); - } - onSuccess?.(); + onSuccess: data => { + queryClient.invalidateQueries(['getUser']); + if (onSuccess) onSuccess(data); }, }, ); From f233006846b73594ed9e32496563235fc07861c7 Mon Sep 17 00:00:00 2001 From: chris-bes Date: Fri, 3 Oct 2025 08:22:09 +1000 Subject: [PATCH 05/26] fix more --- packages/access-policy/package.json | 3 +- .../importStriveLabResults.js | 1 - .../SurveyResponse/SurveyResponse.test.js | 2 +- .../User/addRecentEntities.test.js | 2 + packages/datatrak-web/jest.setup.js | 41 +++++++++++-------- .../src/sync/CentralSyncManager.ts | 4 +- packages/sync/src/utils/getDependencyOrder.ts | 3 +- yarn.lock | 3 +- 8 files changed, 35 insertions(+), 24 deletions(-) diff --git a/packages/access-policy/package.json b/packages/access-policy/package.json index 1e97babd77..f6e62e6d84 100644 --- a/packages/access-policy/package.json +++ b/packages/access-policy/package.json @@ -28,6 +28,7 @@ "rimraf": "^6.0.1" }, "dependencies": { - "@tupaia/constants": "workspace:*" + "@tupaia/constants": "workspace:*", + "@tupaia/utils": "workspace:*" } } diff --git a/packages/central-server/src/apiV2/import/importStriveLabResults/importStriveLabResults.js b/packages/central-server/src/apiV2/import/importStriveLabResults/importStriveLabResults.js index 2fe869522f..93937e64f2 100644 --- a/packages/central-server/src/apiV2/import/importStriveLabResults/importStriveLabResults.js +++ b/packages/central-server/src/apiV2/import/importStriveLabResults/importStriveLabResults.js @@ -3,7 +3,6 @@ import xlsx from 'xlsx'; import { mapKeys, respond, WorkBookParser, UploadError } from '@tupaia/utils'; import { SurveyResponseImporter } from '../../utilities'; import SURVEYS from './surveys.json'; -import { assertCanImportSurveyResponses } from '../importSurveyResponses/assertCanImportSurveyResponses'; import { assertAnyPermissions, assertBESAdminAccess } from '../../../permissions'; const ENTITY_CODE_KEY = 'entityCode'; diff --git a/packages/database/src/__tests__/modelClasses/SurveyResponse/SurveyResponse.test.js b/packages/database/src/__tests__/modelClasses/SurveyResponse/SurveyResponse.test.js index 369f8d104c..775015514c 100644 --- a/packages/database/src/__tests__/modelClasses/SurveyResponse/SurveyResponse.test.js +++ b/packages/database/src/__tests__/modelClasses/SurveyResponse/SurveyResponse.test.js @@ -30,7 +30,7 @@ describe('getLeaderboardQuery()', () => { GROUP BY user_id ) r JOIN user_account ON user_account.id = r.user_id - WHERE email NOT IN (${[...SYSTEM_USERS, ...USERS_EXCLUDED_FROM_LEADER_BOARD].join(', ')}) + WHERE email NOT IN (${[...SYSTEM_USERS, ...USERS_EXCLUDED_FROM_LEADER_BOARD].join(',')}) AND email NOT LIKE '%@beyondessential.com.au' AND email NOT LIKE '%@bes.au' ORDER BY coconuts DESC LIMIT ?;`; diff --git a/packages/database/src/__tests__/modelClasses/User/addRecentEntities.test.js b/packages/database/src/__tests__/modelClasses/User/addRecentEntities.test.js index 6a52351cf9..9059594374 100644 --- a/packages/database/src/__tests__/modelClasses/User/addRecentEntities.test.js +++ b/packages/database/src/__tests__/modelClasses/User/addRecentEntities.test.js @@ -6,11 +6,13 @@ const mockEntities = [...Array(NUM_MOCK_ENTITIES).keys()].flatMap(x => [ id: `DL_${x}`, country_code: 'DL', type: x > NUM_MOCK_ENTITIES / 2 ? 'facility' : 'district', + isProject: () => false, }, { id: `FJ_${x}`, country_code: 'FJ', type: x > NUM_MOCK_ENTITIES / 2 ? 'facility' : 'district', + isProject: () => false, }, ]); const mockUser = { diff --git a/packages/datatrak-web/jest.setup.js b/packages/datatrak-web/jest.setup.js index 462aedf501..3f2e5e065d 100644 --- a/packages/datatrak-web/jest.setup.js +++ b/packages/datatrak-web/jest.setup.js @@ -16,31 +16,38 @@ Object.defineProperty(window, 'matchMedia', { })), }); +const mockModels = { + localSystemFact: { + get: jest.fn().mockResolvedValue('test-device-id'), + set: jest.fn().mockResolvedValue(undefined), + addProjectForSync: jest.fn(), + }, + closeDatabaseConnections: jest.fn(), +}; + +jest.mock('./src/api/CurrentUserContext', () => { + const actual = jest.requireActual('./src/api/CurrentUserContext'); + + return { + ...actual, + useCurrentUserContext: jest.fn(() => ({ + ...actual.useCurrentUserContext(), // Get the actual return value + accessPolicy: {}, // Override just this property + })), + }; +}); + // TODO: Set up database for testing later jest.mock('./src/api/DatabaseContext', () => { const React = require('react'); return { DatabaseContext: React.createContext({ - models: { - localSystemFact: { - get: jest.fn().mockResolvedValue('test-device-id'), - set: jest.fn().mockResolvedValue(undefined), - addProjectForSync: jest.fn(), - }, - closeDatabaseConnections: jest.fn(), - }, + models: mockModels, }), DatabaseProvider: ({ children }) => children, useDatabaseContext: () => ({ - models: { - localSystemFact: { - get: jest.fn().mockResolvedValue('test-device-id'), - set: jest.fn().mockResolvedValue(undefined), - addProjectForSync: jest.fn(), - }, - closeDatabaseConnections: jest.fn(), - }, + models: mockModels, }), }; }); @@ -61,4 +68,4 @@ jest.mock('./src/api/SyncContext', () => { }, }), }; -}); \ No newline at end of file +}); diff --git a/packages/sync-server/src/sync/CentralSyncManager.ts b/packages/sync-server/src/sync/CentralSyncManager.ts index b00c2b4ca5..3f8e581269 100644 --- a/packages/sync-server/src/sync/CentralSyncManager.ts +++ b/packages/sync-server/src/sync/CentralSyncManager.ts @@ -398,7 +398,7 @@ export class CentralSyncManager { // regular changes if (existingProjectIds.length > 0) { - log.info('Snapshotting existing projectssss', { + log.info('Snapshotting existing projects', { existingProjectIds, }); await snapshotOutgoingChanges( @@ -414,7 +414,7 @@ export class CentralSyncManager { // full changes if (newProjectIds.length > 0) { - log.info('Snapshotting new projectsss', { + log.info('Snapshotting new projects', { newProjectIds, }); await snapshotOutgoingChanges( diff --git a/packages/sync/src/utils/getDependencyOrder.ts b/packages/sync/src/utils/getDependencyOrder.ts index 6377d22e47..ce644c2be7 100644 --- a/packages/sync/src/utils/getDependencyOrder.ts +++ b/packages/sync/src/utils/getDependencyOrder.ts @@ -64,5 +64,6 @@ export const sortModelsByDependencyOrder = async ( return orderedDependencies .filter(dep => recordNames.has(dep)) - .map(dep => models.find(r => r.databaseRecord === dep)) as DatabaseModel[]; + .map(dep => models.find(r => r.databaseRecord === dep)) + .filter(m => m !== undefined); // Boolean does not work here. }; diff --git a/yarn.lock b/yarn.lock index 2b20d15950..eaf9cb8500 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12109,6 +12109,7 @@ __metadata: dependencies: "@babel/eslint-parser": ^7.27.5 "@tupaia/constants": "workspace:*" + "@tupaia/utils": "workspace:^" eslint: ^7.32.0 npm-run-all: ^4.1.5 rimraf: ^6.0.1 @@ -13292,7 +13293,7 @@ __metadata: languageName: unknown linkType: soft -"@tupaia/utils@workspace:*, @tupaia/utils@workspace:packages/utils": +"@tupaia/utils@workspace:*, @tupaia/utils@workspace:^, @tupaia/utils@workspace:packages/utils": version: 0.0.0-use.local resolution: "@tupaia/utils@workspace:packages/utils" dependencies: From bdab5cff71954c9a5d81e60e40bb13817ac57f54 Mon Sep 17 00:00:00 2001 From: chris-bes Date: Fri, 3 Oct 2025 08:27:08 +1000 Subject: [PATCH 06/26] fix yarn.lock --- yarn.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yarn.lock b/yarn.lock index eaf9cb8500..2b32fa2517 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12109,7 +12109,7 @@ __metadata: dependencies: "@babel/eslint-parser": ^7.27.5 "@tupaia/constants": "workspace:*" - "@tupaia/utils": "workspace:^" + "@tupaia/utils": "workspace:*" eslint: ^7.32.0 npm-run-all: ^4.1.5 rimraf: ^6.0.1 @@ -13293,7 +13293,7 @@ __metadata: languageName: unknown linkType: soft -"@tupaia/utils@workspace:*, @tupaia/utils@workspace:^, @tupaia/utils@workspace:packages/utils": +"@tupaia/utils@workspace:*, @tupaia/utils@workspace:packages/utils": version: 0.0.0-use.local resolution: "@tupaia/utils@workspace:packages/utils" dependencies: From c2fa96165101adfcd7ecddf36a0939461841b0f9 Mon Sep 17 00:00:00 2001 From: chris-bes Date: Fri, 3 Oct 2025 08:34:11 +1000 Subject: [PATCH 07/26] fixed --- .../src/__tests__/__integration__/login.test.tsx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/datatrak-web/src/__tests__/__integration__/login.test.tsx b/packages/datatrak-web/src/__tests__/__integration__/login.test.tsx index de6f655894..9aa5c627c2 100644 --- a/packages/datatrak-web/src/__tests__/__integration__/login.test.tsx +++ b/packages/datatrak-web/src/__tests__/__integration__/login.test.tsx @@ -40,7 +40,7 @@ describe('Login', () => { renderPage('/login'); expect(await screen.findByRole('heading', { level: 2 })).toHaveTextContent('Log in'); await doLogin(); - server.use(mockUserRequest({ email: 'john@gmail.com', acccessPolicy: [] })); + server.use(mockUserRequest({ email: 'john@gmail.com', accessPolicy: [] })); expect(await screen.findByRole('heading', { level: 1 })).toHaveTextContent(/Select project/i); }); @@ -49,7 +49,7 @@ describe('Login', () => { renderPage('/login'); expect(await screen.findByRole('heading', { level: 2 })).toHaveTextContent('Log in'); - server.use(mockUserRequest({ email: 'john@gmail.com', projectId: 'foo', acccessPolicy: [] })); + server.use(mockUserRequest({ email: 'john@gmail.com', projectId: 'foo', accessPolicy: [] })); await doLogin(); await screen.findByText(/Select survey/i); @@ -59,7 +59,7 @@ describe('Login', () => { renderPage('/survey'); expect(await screen.findByRole('heading', { level: 2 })).toHaveTextContent('Log in'); - server.use(mockUserRequest({ email: 'john@gmail.com', projectId: 'foo', acccessPolicy: [] })); + server.use(mockUserRequest({ email: 'john@gmail.com', projectId: 'foo', accessPolicy: [] })); await doLogin(); expect(await screen.findByRole('heading', { level: 1 })).toHaveTextContent(/Select survey/i); From 1f01d5f67d2aff0ff7f135068be214a3dee3a2d8 Mon Sep 17 00:00:00 2001 From: chris-bes Date: Fri, 3 Oct 2025 09:43:46 +1000 Subject: [PATCH 08/26] fixed type --- packages/sync/src/utils/getDependencyOrder.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/sync/src/utils/getDependencyOrder.ts b/packages/sync/src/utils/getDependencyOrder.ts index ce644c2be7..5b2089c11b 100644 --- a/packages/sync/src/utils/getDependencyOrder.ts +++ b/packages/sync/src/utils/getDependencyOrder.ts @@ -65,5 +65,5 @@ export const sortModelsByDependencyOrder = async ( return orderedDependencies .filter(dep => recordNames.has(dep)) .map(dep => models.find(r => r.databaseRecord === dep)) - .filter(m => m !== undefined); // Boolean does not work here. + .filter((model): model is DatabaseModel => model !== undefined); // Boolean does not work here. }; From 4558f040eec502f81147aa372618e8d4100c9670 Mon Sep 17 00:00:00 2001 From: chris-bes Date: Fri, 3 Oct 2025 11:02:27 +1000 Subject: [PATCH 09/26] fixed test --- .../User/addRecentEntities.test.js | 107 ++++++++---------- .../modelClasses/User/addRecentEntities.js | 21 ++-- 2 files changed, 55 insertions(+), 73 deletions(-) diff --git a/packages/database/src/__tests__/modelClasses/User/addRecentEntities.test.js b/packages/database/src/__tests__/modelClasses/User/addRecentEntities.test.js index 9059594374..e8c43cf211 100644 --- a/packages/database/src/__tests__/modelClasses/User/addRecentEntities.test.js +++ b/packages/database/src/__tests__/modelClasses/User/addRecentEntities.test.js @@ -1,87 +1,76 @@ +import { getTestModels, upsertDummyRecord } from '../../../server'; import { addRecentEntities } from '../../../core/modelClasses/User/addRecentEntities'; const NUM_MOCK_ENTITIES = 10; -const mockEntities = [...Array(NUM_MOCK_ENTITIES).keys()].flatMap(x => [ - { - id: `DL_${x}`, - country_code: 'DL', - type: x > NUM_MOCK_ENTITIES / 2 ? 'facility' : 'district', - isProject: () => false, - }, - { - id: `FJ_${x}`, - country_code: 'FJ', - type: x > NUM_MOCK_ENTITIES / 2 ? 'facility' : 'district', - isProject: () => false, - }, -]); -const mockUser = { - preferences: {}, -}; - -const mockModels = { - database: { executeSql: () => {} }, - user: { - findById: _id => mockUser, - updateById: (_id, update) => { - mockUser.preferences = update.preferences; - }, - }, - entity: { - findById: id => mockEntities.find(x => x.id === id) ?? null, - findManyById: ids => mockEntities.filter(mockEntity => ids.includes(mockEntity.id)), - }, -}; +const USER_ID = 'user'; describe('addRecentEntities', () => { - afterEach(() => { + let mockEntities; + let models; + beforeAll(async () => { + models = getTestModels(); + mockEntities = await Promise.all( + [...Array(NUM_MOCK_ENTITIES).keys()].flatMap(async x => { + const entity1 = await upsertDummyRecord(models.entity, { + id: `DL_${x}`, + country_code: 'DL', + type: x > NUM_MOCK_ENTITIES / 2 ? 'facility' : 'district', + }); + const entity2 = await upsertDummyRecord(models.entity, { + id: `FJ_${x}`, + country_code: 'FJ', + type: x > NUM_MOCK_ENTITIES / 2 ? 'facility' : 'district', + },); + return [entity1, entity2]; + }), + ); + await upsertDummyRecord(models.user, { id: USER_ID }); + }); + + beforeEach(async () => { // Reset the mockUser - mockUser.preferences = {}; + await models.user.updateById(USER_ID, { preferences: {} }); }); it('Adds an entry to the recent entities list', async () => { - await addRecentEntities(mockModels, 'user', ['DL_1']); - expect(mockUser).toMatchObject({ - preferences: { recent_entities: { DL: { district: ['DL_1'] } } }, + await addRecentEntities(models, USER_ID, ['DL_1']); + expect((await models.user.findById(USER_ID)).preferences).toMatchObject({ + recent_entities: { DL: { district: ['DL_1'] } }, }); }); it('Moves an entry to the top of the recent entities list if it already exists in the list', async () => { - await addRecentEntities(mockModels, 'user', ['DL_3', 'DL_2', 'DL_1']); - await addRecentEntities(mockModels, 'user', ['DL_3', 'DL_4']); - expect(mockUser).toMatchObject({ - preferences: { recent_entities: { DL: { district: ['DL_4', 'DL_3', 'DL_1'] } } }, + await addRecentEntities(models, USER_ID, ['DL_3', 'DL_2', 'DL_1']); // 1, 2, 3 + await addRecentEntities(models, USER_ID, ['DL_3', 'DL_4']); // 4, 3 1 + expect((await models.user.findById(USER_ID)).preferences).toMatchObject({ + recent_entities: { DL: { district: ['DL_4', 'DL_3', 'DL_1'] } }, }); }); it('Adds multiple entries to the recent entities list if it already exists in the list', async () => { - await addRecentEntities(mockModels, 'user', ['DL_1', 'DL_2', 'DL_3']); - expect(mockUser).toMatchObject({ - preferences: { recent_entities: { DL: { district: ['DL_3', 'DL_2', 'DL_1'] } } }, + await addRecentEntities(models, USER_ID, ['DL_1', 'DL_2', 'DL_3']); + expect((await models.user.findById(USER_ID)).preferences).toMatchObject({ + recent_entities: { DL: { district: ['DL_3', 'DL_2', 'DL_1'] } }, }); }); it('Cycles out the last entry when exceeded MAX = 3', async () => { - await addRecentEntities(mockModels, 'user', ['DL_1', 'DL_2', 'DL_3', 'DL_4', 'DL_5']); - expect(mockUser).toMatchObject({ - preferences: { recent_entities: { DL: { district: ['DL_5', 'DL_4', 'DL_3'] } } }, + await addRecentEntities(models, USER_ID, ['DL_1', 'DL_2', 'DL_3', 'DL_4', 'DL_5']); + expect((await models.user.findById(USER_ID)).preferences).toMatchObject({ + recent_entities: { DL: { district: ['DL_5', 'DL_4', 'DL_3'] } }, }); }); it('Separately stores lists for different entity types', async () => { - await addRecentEntities(mockModels, 'user', ['DL_1', 'DL_2', 'DL_3', 'DL_6', 'DL_7', 'DL_8']); - expect(mockUser).toMatchObject({ - preferences: { - recent_entities: { - DL: { district: ['DL_3', 'DL_2', 'DL_1'], facility: ['DL_8', 'DL_7', 'DL_6'] }, - }, + await addRecentEntities(models, USER_ID, ['DL_1', 'DL_2', 'DL_3', 'DL_6', 'DL_7', 'DL_8']); + expect((await models.user.findById(USER_ID)).preferences).toMatchObject({ + recent_entities: { + DL: { district: ['DL_3', 'DL_2', 'DL_1'], facility: ['DL_8', 'DL_7', 'DL_6'] }, }, }); }); it('Separately stores lists for different countries', async () => { - await addRecentEntities(mockModels, 'user', ['DL_1', 'DL_2', 'DL_3', 'FJ_1', 'FJ_2', 'FJ_3']); - expect(mockUser).toMatchObject({ - preferences: { - recent_entities: { - DL: { district: ['DL_3', 'DL_2', 'DL_1'] }, - FJ: { district: ['FJ_3', 'FJ_2', 'FJ_1'] }, - }, + await addRecentEntities(models, USER_ID, ['DL_1', 'DL_2', 'DL_3', 'FJ_1', 'FJ_2', 'FJ_3']); + expect((await models.user.findById(USER_ID)).preferences).toMatchObject({ + recent_entities: { + DL: { district: ['DL_3', 'DL_2', 'DL_1'] }, + FJ: { district: ['FJ_3', 'FJ_2', 'FJ_1'] }, }, }); }); diff --git a/packages/database/src/core/modelClasses/User/addRecentEntities.js b/packages/database/src/core/modelClasses/User/addRecentEntities.js index dffdd4eb4d..e858cf3308 100644 --- a/packages/database/src/core/modelClasses/User/addRecentEntities.js +++ b/packages/database/src/core/modelClasses/User/addRecentEntities.js @@ -1,4 +1,3 @@ -import { difference } from 'lodash'; import { ensure } from '@tupaia/tsutils'; import { DatabaseError } from '@tupaia/utils'; @@ -19,17 +18,6 @@ export async function addRecentEntities(models, userId, entityIds) { throw new Error('Usage error: addRecentEntities should not be called with the public user'); } - /** @type {import('../Entity').EntityRecord[]} */ - const entities = await models.entity.findManyById(entityIds); - - if (entityIds.length !== entities.length) { - const diff = difference( - entityIds, - entities.map(e => e.id), - ); - throw new DatabaseError(`Couldn’t find entities with IDs: ${diff.join(', ')}`); - } - /** * @typedef {import('@tupaia/types').Country["code"]} CountryCode * @typedef {import('@tupaia/types').Entity["type"]} EntityType @@ -38,7 +26,12 @@ export async function addRecentEntities(models, userId, entityIds) { */ const recentEntityIds = user.preferences.recent_entities ?? {}; - for (const entity of entities) { + for (const entityId of entityIds) { + const entity = await models.entity.findById(entityId); + if (!entity) { + throw new DatabaseError(`Entity ${entityId} does not exist`); + } + if (entity.isProject()) { // Projects shouldn’t be added to a user’s recent entities throw new Error('addRecentEntities improperly called with a ‘project’-type entity'); @@ -50,7 +43,7 @@ export async function addRecentEntities(models, userId, entityIds) { ); } - const { country_code: countryCode, id: entityId, type: entityType } = entity; + const { country_code: countryCode, type: entityType } = entity; recentEntityIds[countryCode] ??= {}; recentEntityIds[countryCode][entityType] ??= []; recentEntityIds[countryCode][entityType] = recentEntityIds[countryCode][entityType] From 22cb4627856a34a26bb8501399a9fb988dfcc14d Mon Sep 17 00:00:00 2001 From: chris-bes Date: Fri, 3 Oct 2025 11:04:27 +1000 Subject: [PATCH 10/26] fix test --- packages/datatrak-web/jest.setup.js | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/packages/datatrak-web/jest.setup.js b/packages/datatrak-web/jest.setup.js index 3f2e5e065d..fd769bfe2a 100644 --- a/packages/datatrak-web/jest.setup.js +++ b/packages/datatrak-web/jest.setup.js @@ -25,6 +25,27 @@ const mockModels = { closeDatabaseConnections: jest.fn(), }; +jest.mock('@tupaia/database', () => ({ + migrate: jest.fn(), + ModelRegistry: jest.fn().mockImplementation(() => ({})), +})); + +jest.mock('./src/database/createDatabase', () => ({ + createDatabase: jest.fn().mockImplementation(() => ({ + models: { + localSystemFact: { + get: jest.fn().mockImplementation(arg => { + if (arg === 'deviceId') { + return 'test-device-id'; + } + return undefined; + }), + addProjectForSync: jest.fn(), + }, + }, + })), +})); + jest.mock('./src/api/CurrentUserContext', () => { const actual = jest.requireActual('./src/api/CurrentUserContext'); From fb13d035c19211dfba174888686f0d558abb240c Mon Sep 17 00:00:00 2001 From: chris-bes Date: Fri, 3 Oct 2025 12:26:25 +1000 Subject: [PATCH 11/26] fixed undefined import --- .../importSurveyResponses/importSurveyResponses.js | 1 - .../src/apiV2/meditrakApp/postChanges.js | 3 +-- .../apiV2/surveyResponses/ResubmitSurveyResponse.js | 3 +-- .../apiV2/surveyResponses/SubmitSurveyResponses.js | 1 - .../assertCanImportSurveyResponses.test.js | 13 +++++-------- .../core/modelClasses/SurveyResponse/leaderboard.js | 2 +- 6 files changed, 8 insertions(+), 15 deletions(-) diff --git a/packages/central-server/src/apiV2/import/importSurveyResponses/importSurveyResponses.js b/packages/central-server/src/apiV2/import/importSurveyResponses/importSurveyResponses.js index 8b208e1984..9b7e5d63cf 100644 --- a/packages/central-server/src/apiV2/import/importSurveyResponses/importSurveyResponses.js +++ b/packages/central-server/src/apiV2/import/importSurveyResponses/importSurveyResponses.js @@ -25,7 +25,6 @@ import { } from '../../export/exportSurveyResponses'; import { getArrayQueryParameter } from '../../utilities'; import { SurveyResponseUpdatePersistor } from './SurveyResponseUpdatePersistor'; -import { assertCanImportSurveyResponses } from './assertCanImportSurveyResponses'; import { getFailureMessage } from './getFailureMessage'; const ANSWER_TRANSFORMERS = { diff --git a/packages/central-server/src/apiV2/meditrakApp/postChanges.js b/packages/central-server/src/apiV2/meditrakApp/postChanges.js index a841cf1466..f81ec9fe4c 100644 --- a/packages/central-server/src/apiV2/meditrakApp/postChanges.js +++ b/packages/central-server/src/apiV2/meditrakApp/postChanges.js @@ -15,7 +15,6 @@ import { constructIsValidEntityType, } from '@tupaia/utils'; import { updateOrCreateSurveyResponse, addSurveyImage, addSurveyFile } from './utilities'; -import { assertCanSubmitSurveyResponses } from '../import/importSurveyResponses/assertCanImportSurveyResponses'; import { assertAnyPermissions, assertBESAdminAccess } from '../../permissions'; import { translateObjectFields, @@ -59,7 +58,7 @@ export async function postChanges(req, res) { .filter(c => c.action === ACTIONS.SubmitSurveyResponse) .map(c => c.translatedPayload.survey_response || c.translatedPayload); const surveyResponsePermissionsChecker = async accessPolicy => { - await assertCanSubmitSurveyResponses(accessPolicy, transactingModels, surveyResponsePayloads); + await transactingModels.surveyResponse.assertCanSubmit(accessPolicy, surveyResponsePayloads); }; await req.assertPermissions( assertAnyPermissions([assertBESAdminAccess, surveyResponsePermissionsChecker]), diff --git a/packages/central-server/src/apiV2/surveyResponses/ResubmitSurveyResponse.js b/packages/central-server/src/apiV2/surveyResponses/ResubmitSurveyResponse.js index 4afdd41637..34bba0915e 100644 --- a/packages/central-server/src/apiV2/surveyResponses/ResubmitSurveyResponse.js +++ b/packages/central-server/src/apiV2/surveyResponses/ResubmitSurveyResponse.js @@ -8,7 +8,6 @@ import { assertAnyPermissions, assertBESAdminAccess, } from '../../permissions'; -import { assertCanSubmitSurveyResponses } from '../import/importSurveyResponses/assertCanImportSurveyResponses'; import { RouteHandler } from '../RouteHandler'; import { assertSurveyResponsePermissions } from './assertSurveyResponsePermissions'; @@ -39,7 +38,7 @@ export class ResubmitSurveyResponse extends RouteHandler { ); const newSurveyResponsePermissionsChecker = async accessPolicy => { - await assertCanSubmitSurveyResponses(accessPolicy, transactingModels, [ + await transactingModels.surveyResponse.assertCanSubmit(accessPolicy, [ this.newSurveyResponse, ]); }; diff --git a/packages/central-server/src/apiV2/surveyResponses/SubmitSurveyResponses.js b/packages/central-server/src/apiV2/surveyResponses/SubmitSurveyResponses.js index 4ec41e8cf1..9fa0a57bd6 100644 --- a/packages/central-server/src/apiV2/surveyResponses/SubmitSurveyResponses.js +++ b/packages/central-server/src/apiV2/surveyResponses/SubmitSurveyResponses.js @@ -2,7 +2,6 @@ import { AnalyticsRefresher, SurveyResponseModel } from '@tupaia/database'; import { respond } from '@tupaia/utils'; import { ANSWER_BODY_PARSERS } from '../../dataAccessors'; import { assertAnyPermissions, assertBESAdminAccess } from '../../permissions'; -import { assertCanSubmitSurveyResponses } from '../import/importSurveyResponses/assertCanImportSurveyResponses'; import { RouteHandler } from '../RouteHandler'; export class SubmitSurveyResponses extends RouteHandler { diff --git a/packages/central-server/src/tests/apiV2/import/importSurveyResponses/assertCanImportSurveyResponses.test.js b/packages/central-server/src/tests/apiV2/import/importSurveyResponses/assertCanImportSurveyResponses.test.js index 2d469a07f1..aeb28d7135 100644 --- a/packages/central-server/src/tests/apiV2/import/importSurveyResponses/assertCanImportSurveyResponses.test.js +++ b/packages/central-server/src/tests/apiV2/import/importSurveyResponses/assertCanImportSurveyResponses.test.js @@ -8,7 +8,6 @@ import { } from '@tupaia/database'; import { TUPAIA_ADMIN_PANEL_PERMISSION_GROUP } from '../../../../permissions'; import { getModels } from '../../../testUtilities'; -import { assertCanImportSurveyResponses } from '../../../../apiV2/import/importSurveyResponses/assertCanImportSurveyResponses'; const DEFAULT_POLICY = { DL: ['Public'], @@ -91,9 +90,8 @@ describe('assertCanImportSurveyResponses(): Permissions checker for Importing Su const entitiesBySurveyCode = { [SURVEY_CODE_1]: ['KI_1_test', 'KI_2_test', 'KI_3_test'], }; - const result = await assertCanImportSurveyResponses( + const result = await models.surveyResponse.assertCanImport( defaultAccessPolicy, - models, entitiesBySurveyCode, ); @@ -106,9 +104,8 @@ describe('assertCanImportSurveyResponses(): Permissions checker for Importing Su [SURVEY_CODE_1]: ['KI_1_test', 'KI_2_test', 'KI_3_test'], [SURVEY_CODE_2]: ['VU_1_test', 'VU_2_test', 'VU_3_test'], }; - const result = await assertCanImportSurveyResponses( + const result = await models.surveyResponse.assertCanImport( defaultAccessPolicy, - models, entitiesBySurveyCode, ); @@ -121,7 +118,7 @@ describe('assertCanImportSurveyResponses(): Permissions checker for Importing Su [SURVEY_CODE_2]: ['LA_1_test', 'VU_2_test', 'VU_3_test'], }; - expect(() => assertCanImportSurveyResponses(defaultAccessPolicy, models, entitiesBySurveyCode)) + expect(() => models.surveyResponse.assertCanImport(defaultAccessPolicy, entitiesBySurveyCode)) .to.throw; }); @@ -140,7 +137,7 @@ describe('assertCanImportSurveyResponses(): Permissions checker for Importing Su [SURVEY_CODE_2]: ['VU_1_test', 'VU_2_test', 'VU_3_test'], }; - expect(() => assertCanImportSurveyResponses(accessPolicy, models, entitiesBySurveyCode)).to + expect(() => models.surveyResponse.assertCanImport(accessPolicy, entitiesBySurveyCode)).to .throw; }); @@ -160,7 +157,7 @@ describe('assertCanImportSurveyResponses(): Permissions checker for Importing Su [SURVEY_CODE_2]: ['VU_1_test', 'VU_2_test', 'VU_3_test'], }; - expect(() => assertCanImportSurveyResponses(accessPolicy, models, entitiesBySurveyCode)).to + expect(() => models.surveyResponse.assertCanImport(accessPolicy, entitiesBySurveyCode)).to .throw; }); }); diff --git a/packages/database/src/core/modelClasses/SurveyResponse/leaderboard.js b/packages/database/src/core/modelClasses/SurveyResponse/leaderboard.js index 338a1f47ea..274205e408 100644 --- a/packages/database/src/core/modelClasses/SurveyResponse/leaderboard.js +++ b/packages/database/src/core/modelClasses/SurveyResponse/leaderboard.js @@ -32,7 +32,7 @@ export function getLeaderboardQuery(projectId = '') { return ` SELECT r.user_id, user_account.first_name, user_account.last_name, r.coconuts, r.pigs FROM ( - SELECT user_id, COUNT(*)::INT as coconuts, FLOOR(COUNT(*) / 100)::INT AS pigs + SELECT user_id, COUNT(*)::INT AS coconuts, FLOOR(COUNT(*) / 100)::INT AS pigs FROM survey_response JOIN survey ON survey.id=survey_id ${projectId ? 'WHERE survey.project_id = ?' : ''} From d86246963144d5304b04f64f630dce27ab3c134d Mon Sep 17 00:00:00 2001 From: Jasper Lai <33956381+jaskfla@users.noreply.github.com> Date: Fri, 3 Oct 2025 16:04:45 +1300 Subject: [PATCH 12/26] fix mocked `/getUesr` response --- .../src/__tests__/__integration__/survey.test.tsx | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/datatrak-web/src/__tests__/__integration__/survey.test.tsx b/packages/datatrak-web/src/__tests__/__integration__/survey.test.tsx index dfd7a01c1d..4e36eed920 100644 --- a/packages/datatrak-web/src/__tests__/__integration__/survey.test.tsx +++ b/packages/datatrak-web/src/__tests__/__integration__/survey.test.tsx @@ -9,7 +9,14 @@ import { handlers } from '../mocks/handlers'; const server = setupServer( ...handlers, rest.get('*/v1/getUser', (_, res, ctx) => { - return res(ctx.status(200), ctx.json({ name: 'John Smith', email: 'john@gmail.com' })); + return res( + ctx.status(200), + ctx.json({ + name: 'John Smith', + email: 'john@gmail.com', + id: '0'.repeat(24), + }), + ); }), rest.get('*/v1/*', (_, res, ctx) => { return res(ctx.status(200), ctx.json([])); From ad87c496aa39726dc565dcf15d8685340053e806 Mon Sep 17 00:00:00 2001 From: chris-bes Date: Fri, 3 Oct 2025 13:33:47 +1000 Subject: [PATCH 13/26] fixed test --- .../modelClasses/SurveyResponse/SurveyResponse.test.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/database/src/__tests__/modelClasses/SurveyResponse/SurveyResponse.test.js b/packages/database/src/__tests__/modelClasses/SurveyResponse/SurveyResponse.test.js index 775015514c..a8b49cb256 100644 --- a/packages/database/src/__tests__/modelClasses/SurveyResponse/SurveyResponse.test.js +++ b/packages/database/src/__tests__/modelClasses/SurveyResponse/SurveyResponse.test.js @@ -23,7 +23,7 @@ describe('getLeaderboardQuery()', () => { const expectedLeaderboard = ` SELECT r.user_id, user_account.first_name, user_account.last_name, r.coconuts, r.pigs FROM ( - SELECT user_id, COUNT(*)::INT AS coconuts, FLOOR(COUNT(*) / 100)::INT as pigs + SELECT user_id, COUNT(*)::INT AS coconuts, FLOOR(COUNT(*) / 100)::INT AS pigs FROM survey_response JOIN survey ON survey.id=survey_id WHERE survey.project_id = ? @@ -47,7 +47,7 @@ describe('getLeaderboardQuery()', () => { const expectedLeaderboard = ` SELECT r.user_id, user_account.first_name, user_account.last_name, r.coconuts, r.pigs FROM ( - SELECT user_id, COUNT(*)::INT as coconuts, FLOOR(COUNT(*) / 100)::INT AS pigs + SELECT user_id, COUNT(*)::INT AS coconuts, FLOOR(COUNT(*) / 100)::INT AS pigs FROM survey_response JOIN survey ON survey.id=survey_id WHERE survey.project_id = ? From 66358a4c0f818335ce79fa849680dfc00c8c728d Mon Sep 17 00:00:00 2001 From: chris-bes Date: Fri, 3 Oct 2025 13:37:46 +1000 Subject: [PATCH 14/26] fixed test --- .../__tests__/features/Survey/CopySurveyUrlButton.test.tsx | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/datatrak-web/src/__tests__/features/Survey/CopySurveyUrlButton.test.tsx b/packages/datatrak-web/src/__tests__/features/Survey/CopySurveyUrlButton.test.tsx index d3d2205b3e..67e554b0bb 100644 --- a/packages/datatrak-web/src/__tests__/features/Survey/CopySurveyUrlButton.test.tsx +++ b/packages/datatrak-web/src/__tests__/features/Survey/CopySurveyUrlButton.test.tsx @@ -9,7 +9,10 @@ import { handlers } from '../../mocks/handlers'; const server = setupServer( ...handlers, rest.get('*/v1/getUser', (_, res, ctx) => { - return res(ctx.status(200), ctx.json({ name: 'John Smith', email: 'john@gmail.com' })); + return res( + ctx.status(200), + ctx.json({ name: 'John Smith', email: 'john@gmail.com', id: '0'.repeat(24) }), + ); }), ); From 450230a6cf3b2895bb667ebb4b473a7af3b1941a Mon Sep 17 00:00:00 2001 From: chris-bes Date: Fri, 3 Oct 2025 14:52:00 +1000 Subject: [PATCH 15/26] pass models into assertCanSubmit and assertCanImport of SurveyResponse --- .../importStriveLabResults.js | 2 +- .../importSurveyResponses.js | 2 +- .../src/apiV2/meditrakApp/postChanges.js | 6 +++++- .../surveyResponses/ResubmitSurveyResponse.js | 2 +- .../surveyResponses/SubmitSurveyResponses.js | 6 +++++- .../src/database/models/SurveyResponse.js | 4 ++-- .../assertCanImportSurveyResponses.test.js | 8 ++++--- .../SurveyResponse/SurveyResponse.js | 21 +++++++++++-------- 8 files changed, 32 insertions(+), 19 deletions(-) diff --git a/packages/central-server/src/apiV2/import/importStriveLabResults/importStriveLabResults.js b/packages/central-server/src/apiV2/import/importStriveLabResults/importStriveLabResults.js index 93937e64f2..57b72ad7d6 100644 --- a/packages/central-server/src/apiV2/import/importStriveLabResults/importStriveLabResults.js +++ b/packages/central-server/src/apiV2/import/importStriveLabResults/importStriveLabResults.js @@ -86,7 +86,7 @@ export const importStriveLabResults = async (req, res) => { const entitiesGroupedBySurveyCode = await getEntitiesGroupedBySurveyCode(models, inputsPerSurvey); const importSurveyResponsePermissionsChecker = async accessPolicy => { - await models.surveyResponse.assertCanImport(accessPolicy, entitiesGroupedBySurveyCode); + await models.surveyResponse.assertCanImport(models, accessPolicy, entitiesGroupedBySurveyCode); }; await req.assertPermissions( diff --git a/packages/central-server/src/apiV2/import/importSurveyResponses/importSurveyResponses.js b/packages/central-server/src/apiV2/import/importSurveyResponses/importSurveyResponses.js index 9b7e5d63cf..b2b9ebdd71 100644 --- a/packages/central-server/src/apiV2/import/importSurveyResponses/importSurveyResponses.js +++ b/packages/central-server/src/apiV2/import/importSurveyResponses/importSurveyResponses.js @@ -138,7 +138,7 @@ export async function importSurveyResponses(req, res) { const entityCodeToId = reduceToDictionary(entities, 'code', 'id'); const importSurveyResponsePermissionsChecker = async accessPolicy => { - await models.surveyResponse.assertCanImport(accessPolicy, entitiesBySurveyCode); + await models.surveyResponse.assertCanImport(models, accessPolicy, entitiesBySurveyCode); }; await req.assertPermissions( diff --git a/packages/central-server/src/apiV2/meditrakApp/postChanges.js b/packages/central-server/src/apiV2/meditrakApp/postChanges.js index f81ec9fe4c..a599992936 100644 --- a/packages/central-server/src/apiV2/meditrakApp/postChanges.js +++ b/packages/central-server/src/apiV2/meditrakApp/postChanges.js @@ -58,7 +58,11 @@ export async function postChanges(req, res) { .filter(c => c.action === ACTIONS.SubmitSurveyResponse) .map(c => c.translatedPayload.survey_response || c.translatedPayload); const surveyResponsePermissionsChecker = async accessPolicy => { - await transactingModels.surveyResponse.assertCanSubmit(accessPolicy, surveyResponsePayloads); + await transactingModels.surveyResponse.assertCanSubmit( + transactingModels, + accessPolicy, + surveyResponsePayloads, + ); }; await req.assertPermissions( assertAnyPermissions([assertBESAdminAccess, surveyResponsePermissionsChecker]), diff --git a/packages/central-server/src/apiV2/surveyResponses/ResubmitSurveyResponse.js b/packages/central-server/src/apiV2/surveyResponses/ResubmitSurveyResponse.js index 34bba0915e..b9a8cee85f 100644 --- a/packages/central-server/src/apiV2/surveyResponses/ResubmitSurveyResponse.js +++ b/packages/central-server/src/apiV2/surveyResponses/ResubmitSurveyResponse.js @@ -38,7 +38,7 @@ export class ResubmitSurveyResponse extends RouteHandler { ); const newSurveyResponsePermissionsChecker = async accessPolicy => { - await transactingModels.surveyResponse.assertCanSubmit(accessPolicy, [ + await transactingModels.surveyResponse.assertCanSubmit(transactingModels, accessPolicy, [ this.newSurveyResponse, ]); }; diff --git a/packages/central-server/src/apiV2/surveyResponses/SubmitSurveyResponses.js b/packages/central-server/src/apiV2/surveyResponses/SubmitSurveyResponses.js index 9fa0a57bd6..eb657ae1ff 100644 --- a/packages/central-server/src/apiV2/surveyResponses/SubmitSurveyResponses.js +++ b/packages/central-server/src/apiV2/surveyResponses/SubmitSurveyResponses.js @@ -14,7 +14,11 @@ export class SubmitSurveyResponses extends RouteHandler { async assertUserHasAccess(transactingModels) { // Check permissions const surveyResponsePermissionsChecker = async accessPolicy => { - await transactingModels.surveyResponse.assertCanSubmit(accessPolicy, this.surveyResponses); + await transactingModels.surveyResponse.assertCanSubmit( + transactingModels, + accessPolicy, + this.surveyResponses, + ); }; await this.assertPermissions( assertAnyPermissions([assertBESAdminAccess, surveyResponsePermissionsChecker]), diff --git a/packages/central-server/src/database/models/SurveyResponse.js b/packages/central-server/src/database/models/SurveyResponse.js index 596428e218..ab1a3eaf1c 100644 --- a/packages/central-server/src/database/models/SurveyResponse.js +++ b/packages/central-server/src/database/models/SurveyResponse.js @@ -2,7 +2,7 @@ import momentTimezone from 'moment-timezone'; import moment from 'moment'; import { - MaterializedViewLogDatabaseModel, + SurveyResponseModel as BaseSurveyResponseModel, DatabaseRecord, RECORDS, createSurveyResponsePermissionFilter, @@ -58,7 +58,7 @@ class SurveyResponseRecord extends DatabaseRecord { } } -export class SurveyResponseModel extends MaterializedViewLogDatabaseModel { +export class SurveyResponseModel extends BaseSurveyResponseModel { notifiers = [onChangeMarkAnswersChanged]; get DatabaseRecordClass() { diff --git a/packages/central-server/src/tests/apiV2/import/importSurveyResponses/assertCanImportSurveyResponses.test.js b/packages/central-server/src/tests/apiV2/import/importSurveyResponses/assertCanImportSurveyResponses.test.js index aeb28d7135..f3f055dedb 100644 --- a/packages/central-server/src/tests/apiV2/import/importSurveyResponses/assertCanImportSurveyResponses.test.js +++ b/packages/central-server/src/tests/apiV2/import/importSurveyResponses/assertCanImportSurveyResponses.test.js @@ -91,6 +91,7 @@ describe('assertCanImportSurveyResponses(): Permissions checker for Importing Su [SURVEY_CODE_1]: ['KI_1_test', 'KI_2_test', 'KI_3_test'], }; const result = await models.surveyResponse.assertCanImport( + models, defaultAccessPolicy, entitiesBySurveyCode, ); @@ -105,6 +106,7 @@ describe('assertCanImportSurveyResponses(): Permissions checker for Importing Su [SURVEY_CODE_2]: ['VU_1_test', 'VU_2_test', 'VU_3_test'], }; const result = await models.surveyResponse.assertCanImport( + models, defaultAccessPolicy, entitiesBySurveyCode, ); @@ -118,7 +120,7 @@ describe('assertCanImportSurveyResponses(): Permissions checker for Importing Su [SURVEY_CODE_2]: ['LA_1_test', 'VU_2_test', 'VU_3_test'], }; - expect(() => models.surveyResponse.assertCanImport(defaultAccessPolicy, entitiesBySurveyCode)) + expect(() => models.surveyResponse.assertCanImport(models, defaultAccessPolicy, entitiesBySurveyCode)) .to.throw; }); @@ -137,7 +139,7 @@ describe('assertCanImportSurveyResponses(): Permissions checker for Importing Su [SURVEY_CODE_2]: ['VU_1_test', 'VU_2_test', 'VU_3_test'], }; - expect(() => models.surveyResponse.assertCanImport(accessPolicy, entitiesBySurveyCode)).to + expect(() => models.surveyResponse.assertCanImport(models, accessPolicy, entitiesBySurveyCode)).to .throw; }); @@ -157,7 +159,7 @@ describe('assertCanImportSurveyResponses(): Permissions checker for Importing Su [SURVEY_CODE_2]: ['VU_1_test', 'VU_2_test', 'VU_3_test'], }; - expect(() => models.surveyResponse.assertCanImport(accessPolicy, entitiesBySurveyCode)).to + expect(() => models.surveyResponse.assertCanImport(models, accessPolicy, entitiesBySurveyCode)).to .throw; }); }); diff --git a/packages/database/src/core/modelClasses/SurveyResponse/SurveyResponse.js b/packages/database/src/core/modelClasses/SurveyResponse/SurveyResponse.js index 76dc9ee72b..43663bd8ae 100644 --- a/packages/database/src/core/modelClasses/SurveyResponse/SurveyResponse.js +++ b/packages/database/src/core/modelClasses/SurveyResponse/SurveyResponse.js @@ -1,3 +1,4 @@ +import log from 'winston'; import { difference, uniq } from 'es-toolkit'; import { flattenDeep, groupBy, keyBy } from 'es-toolkit/compat'; @@ -96,16 +97,17 @@ export class SurveyResponseModel extends MaterializedViewLogDatabaseModel { } /** + * @param {ModelRegistry} models * @param {import('@tupaia/access-policy').AccessPolicy} accessPolicy * @param {Record} entitiesBySurveyCode * @returns {true} If and only if the assertion passes, otherwise throws. * @throws {PermissionsError} */ - async assertCanImport(accessPolicy, entitiesBySurveyCode) { + async assertCanImport(models, accessPolicy, entitiesBySurveyCode) { const allEntityCodes = flattenDeep(Object.values(entitiesBySurveyCode)); const surveyCodes = Object.keys(entitiesBySurveyCode); - await this.database.wrapInReadOnlyTransaction(async transactingModels => { + await models.wrapInReadOnlyTransaction(async transactingModels => { /** @type {[EntityRecord[], SurveyRecord[]]} */ const [allEntities, surveys] = await Promise.all([ transactingModels.entity.findManyByColumn('code', allEntityCodes), @@ -113,7 +115,7 @@ export class SurveyResponseModel extends MaterializedViewLogDatabaseModel { ]); if (allEntities.some(isNullish)) { - console.error('Unexpected nullish element in `allEntities`', { allEntities }); + log.error('Unexpected nullish element in `allEntities`', { allEntities }); } const codeToSurvey = keyBy(surveys, 'code'); @@ -127,7 +129,7 @@ export class SurveyResponseModel extends MaterializedViewLogDatabaseModel { for (const [surveyCode, entityCodes] of Object.entries(entitiesBySurveyCode)) { const survey = codeToSurvey[surveyCode]; if (isNullish(survey)) { - console.error(`Unexpected nullish survey (code '${surveyCode}')`, { codeToSurvey }); + log.error(`Unexpected nullish survey (code '${surveyCode}')`, { codeToSurvey }); } const responseEntities = allEntities.filter(e => entityCodes.includes(e.code)); @@ -140,7 +142,7 @@ export class SurveyResponseModel extends MaterializedViewLogDatabaseModel { ); if (surveyResponseCountries.some(isNullish)) { - console.error(`Unexpected nullish element in countries for survey ${surveyCode}`, { + log.error(`Unexpected nullish element in countries for survey ${surveyCode}`, { surveyResponseCountries, }); } @@ -169,7 +171,7 @@ export class SurveyResponseModel extends MaterializedViewLogDatabaseModel { const entities = entitiesByCountryCode[surveyResponseCountry.code]; if (entities.some(isNullish)) { - console.error('Unexpected nullish element in `entities`', { entities }); + log.error('Unexpected nullish element in `entities`', { entities }); } const entityCodesString = entities.map(e => e.code).join(', '); @@ -194,12 +196,13 @@ export class SurveyResponseModel extends MaterializedViewLogDatabaseModel { } /** + * @param {ModelRegistry} models * @param {import('@tupaia/access-policy').AccessPolicy} accessPolicy * @param {Array} surveyResponses Assumed to have already been validated. * @returns {true} If and only if the assertion passes, otherwise throws. * @throws {PermissionsError} */ - async assertCanSubmit(accessPolicy, surveyResponses) { + async assertCanSubmit(models, accessPolicy, surveyResponses) { const entitiesBySurveyCode = {}; /** @type {Survey["id"][]} */ @@ -211,7 +214,7 @@ export class SurveyResponseModel extends MaterializedViewLogDatabaseModel { return acc; }, []); - return await this.database.wrapInReadOnlyTransaction(async transactingModels => { + return await models.wrapInReadOnlyTransaction(async transactingModels => { /** @type {SurveyRecord[]} */ const surveys = await transactingModels.survey.findManyById(surveyIds); const surveyCodesById = reduceToDictionary(surveys, 'id', 'code'); @@ -225,7 +228,7 @@ export class SurveyResponseModel extends MaterializedViewLogDatabaseModel { (entitiesBySurveyCode[surveyCode] ??= []).push(entityCode); } - return await this.assertCanImport(accessPolicy, entitiesBySurveyCode); + return await this.assertCanImport(transactingModels, accessPolicy, entitiesBySurveyCode); }); } From 3b82aeaa0478ffa02f3cbe2ce3a81c594ddb25c1 Mon Sep 17 00:00:00 2001 From: chris-bes Date: Fri, 3 Oct 2025 15:53:31 +1000 Subject: [PATCH 16/26] removed unused --- packages/database/src/core/sync/buildSyncLookupSelect.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/database/src/core/sync/buildSyncLookupSelect.js b/packages/database/src/core/sync/buildSyncLookupSelect.js index 649ff9db54..ebcdb2a235 100644 --- a/packages/database/src/core/sync/buildSyncLookupSelect.js +++ b/packages/database/src/core/sync/buildSyncLookupSelect.js @@ -2,7 +2,7 @@ import { COLUMNS_EXCLUDED_FROM_SYNC } from '@tupaia/constants'; export async function buildSyncLookupSelect(model, columns = {}) { const attributes = Object.keys(await model.fetchSchema()); - const { projectIds, userIds } = columns; + const { projectIds } = columns; const table = model.databaseRecord; return ` From 59f92b4ee43951f3fea89732088a135a4c84fa56 Mon Sep 17 00:00:00 2001 From: chris-bes Date: Fri, 3 Oct 2025 18:28:43 +1000 Subject: [PATCH 17/26] fixed missing import --- packages/central-server/src/permissions/assertions.js | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/central-server/src/permissions/assertions.js b/packages/central-server/src/permissions/assertions.js index 5674ff9988..64b9311d46 100644 --- a/packages/central-server/src/permissions/assertions.js +++ b/packages/central-server/src/permissions/assertions.js @@ -1,3 +1,4 @@ +import { ensure } from '@tupaia/tsutils'; /* Re-export for backward compatibility. Prefer importing directly from @tupaia/access-policy. */ export { allowNoPermissions, From 89712269031e2d5926f88c2d33d6fd9f27ac0553 Mon Sep 17 00:00:00 2001 From: chris-bes Date: Sat, 4 Oct 2025 01:44:00 +1000 Subject: [PATCH 18/26] fix permission test --- packages/central-server/src/permissions/assertions.js | 1 + .../CreateMapOverlayVisualisations.test.js | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/central-server/src/permissions/assertions.js b/packages/central-server/src/permissions/assertions.js index 64b9311d46..79dd65bf29 100644 --- a/packages/central-server/src/permissions/assertions.js +++ b/packages/central-server/src/permissions/assertions.js @@ -1,3 +1,4 @@ +import { TUPAIA_ADMIN_PANEL_PERMISSION_GROUP } from '@tupaia/constants'; import { ensure } from '@tupaia/tsutils'; /* Re-export for backward compatibility. Prefer importing directly from @tupaia/access-policy. */ export { diff --git a/packages/central-server/src/tests/apiV2/mapOverlayVisualisations/CreateMapOverlayVisualisations.test.js b/packages/central-server/src/tests/apiV2/mapOverlayVisualisations/CreateMapOverlayVisualisations.test.js index 99c8cfef4d..be8bf42e3a 100644 --- a/packages/central-server/src/tests/apiV2/mapOverlayVisualisations/CreateMapOverlayVisualisations.test.js +++ b/packages/central-server/src/tests/apiV2/mapOverlayVisualisations/CreateMapOverlayVisualisations.test.js @@ -97,7 +97,7 @@ describe('POST map overlay visualisations', async () => { }); expectError( response, - 'Database error: Creating record - You do not have access to all related permission groups', + 'Database error: Creating record - Need access to Test Permission Group', ); }); }); From 34875ca167b6b54ea85f0ea4efdf563d73bac00f Mon Sep 17 00:00:00 2001 From: chris-bes Date: Sat, 4 Oct 2025 22:05:06 +1000 Subject: [PATCH 19/26] fixed undefined --- packages/central-server/src/permissions/assertions.js | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/central-server/src/permissions/assertions.js b/packages/central-server/src/permissions/assertions.js index 79dd65bf29..b93c979028 100644 --- a/packages/central-server/src/permissions/assertions.js +++ b/packages/central-server/src/permissions/assertions.js @@ -1,5 +1,6 @@ import { TUPAIA_ADMIN_PANEL_PERMISSION_GROUP } from '@tupaia/constants'; import { ensure } from '@tupaia/tsutils'; +import { PermissionsError } from '@tupaia/utils'; /* Re-export for backward compatibility. Prefer importing directly from @tupaia/access-policy. */ export { allowNoPermissions, From 872a75ef1a850680a695dd2d554373b1c0ee8775 Mon Sep 17 00:00:00 2001 From: chris-bes Date: Mon, 6 Oct 2025 09:54:36 +1100 Subject: [PATCH 20/26] removed performance --- packages/sync-server/src/sync/CentralSyncManager.ts | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/packages/sync-server/src/sync/CentralSyncManager.ts b/packages/sync-server/src/sync/CentralSyncManager.ts index fc2ca21157..dd462a0dc5 100644 --- a/packages/sync-server/src/sync/CentralSyncManager.ts +++ b/packages/sync-server/src/sync/CentralSyncManager.ts @@ -374,21 +374,10 @@ export class CentralSyncManager { }, snapshotTransactionTimeoutMs); } - performance.mark('start-findLastSuccessfulSyncedProjects'); const lastSuccessfulSyncedProjectIds = await findLastSuccessfulSyncedProjects( transactingModels.database, deviceId, ); - performance.mark('end-findLastSuccessfulSyncedProjects'); - log.info( - 'findLastSuccessfulSyncedProjects', - performance.measure( - 'findLastSuccessfulSyncedProjects', - 'start-findLastSuccessfulSyncedProjects', - 'end-findLastSuccessfulSyncedProjects', - ), - ); - const existingProjectIds = projectIds.filter(projectId => lastSuccessfulSyncedProjectIds.includes(projectId), ); From 46f4730246a58a49976153d441ec1fa35d0e8b26 Mon Sep 17 00:00:00 2001 From: chris-bes Date: Mon, 6 Oct 2025 10:09:20 +1100 Subject: [PATCH 21/26] fixed more --- packages/datatrak-web/src/api/DatabaseContext.tsx | 4 +++- packages/sync/src/utils/saveChanges.ts | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/datatrak-web/src/api/DatabaseContext.tsx b/packages/datatrak-web/src/api/DatabaseContext.tsx index 68a1ad3d77..5f9597b2ea 100644 --- a/packages/datatrak-web/src/api/DatabaseContext.tsx +++ b/packages/datatrak-web/src/api/DatabaseContext.tsx @@ -15,15 +15,17 @@ export const DatabaseProvider = ({ children }: { children: Readonly(null); useEffect(() => { + let modelsInstance: DatatrakWebModelRegistry | null = null; const init = async () => { const { models } = await createDatabase(); + modelsInstance = models; setModels(models); }; init(); return () => { - models?.closeDatabaseConnections(); + modelsInstance?.closeDatabaseConnections(); }; }, []); diff --git a/packages/sync/src/utils/saveChanges.ts b/packages/sync/src/utils/saveChanges.ts index 73c99cee05..4c2165c2ef 100644 --- a/packages/sync/src/utils/saveChanges.ts +++ b/packages/sync/src/utils/saveChanges.ts @@ -18,8 +18,8 @@ export const saveCreates = async ( batch.map(async row => { try { await model.create(row); - } catch (error) { - throw new Error(`Insert failed with '${e.message}', recordId: ${row.id}`); + } catch (error: any) { + throw new Error(`Insert failed with '${error.message}', recordId: ${row.id}`); } }), ); From 503ed8edb2bdfd450a653f6a00ccb3acb3e08c41 Mon Sep 17 00:00:00 2001 From: chris-bes Date: Mon, 6 Oct 2025 10:30:03 +1100 Subject: [PATCH 22/26] fix more bugs --- packages/database/src/core/modelClasses/Entity.js | 9 ++++----- packages/database/src/core/sync/buildSyncLookupSelect.js | 5 ++--- .../src/database/entity/getEntityDescendants.ts | 3 ++- packages/datatrak-web/src/sync/ClientSyncManager.ts | 2 +- packages/sync/src/utils/manageSnapshotTable.ts | 2 +- 5 files changed, 10 insertions(+), 11 deletions(-) diff --git a/packages/database/src/core/modelClasses/Entity.js b/packages/database/src/core/modelClasses/Entity.js index e7b5bba075..62274fd29f 100644 --- a/packages/database/src/core/modelClasses/Entity.js +++ b/packages/database/src/core/modelClasses/Entity.js @@ -336,6 +336,10 @@ export class EntityRecord extends DatabaseRecord { export class EntityModel extends MaterializedViewLogDatabaseModel { static syncDirection = SyncDirections.BIDIRECTIONAL; + get ExcludedFieldsFromSync() { + return ['point', 'bounds', 'region', 'parent_id']; + } + get DatabaseRecordClass() { return EntityRecord; } @@ -699,9 +703,4 @@ export class EntityModel extends MaterializedViewLogDatabaseModel { groupBy: ['entity.id'], }; } - - sanitizeForClient = data => { - const { point, bounds, region, parent_id, ...sanitizedData } = data; - return sanitizedData; - }; } diff --git a/packages/database/src/core/sync/buildSyncLookupSelect.js b/packages/database/src/core/sync/buildSyncLookupSelect.js index ebcdb2a235..7e3a986d22 100644 --- a/packages/database/src/core/sync/buildSyncLookupSelect.js +++ b/packages/database/src/core/sync/buildSyncLookupSelect.js @@ -4,6 +4,7 @@ export async function buildSyncLookupSelect(model, columns = {}) { const attributes = Object.keys(await model.fetchSchema()); const { projectIds } = columns; const table = model.databaseRecord; + const excludedFields = [...(model.ExcludedFieldsFromSync || []), ...COLUMNS_EXCLUDED_FROM_SYNC]; return ` SELECT @@ -12,9 +13,7 @@ export async function buildSyncLookupSelect(model, columns = {}) { COALESCE(:updatedAtSyncTick, ${table}.updated_at_sync_tick), sync_device_tick.device_id, json_build_object( - ${attributes - .filter(a => !COLUMNS_EXCLUDED_FROM_SYNC.includes(a)) - .map(a => `'${a}', ${table}.${a}`)} + ${attributes.filter(a => !excludedFields.includes(a)).map(a => `'${a}', ${table}.${a}`)} ), ${projectIds || 'NULL'} `; diff --git a/packages/datatrak-web/src/database/entity/getEntityDescendants.ts b/packages/datatrak-web/src/database/entity/getEntityDescendants.ts index 59ffc96d12..e03d3dccd4 100644 --- a/packages/datatrak-web/src/database/entity/getEntityDescendants.ts +++ b/packages/datatrak-web/src/database/entity/getEntityDescendants.ts @@ -13,6 +13,7 @@ import { CurrentUser } from '../../api'; import { DatatrakWebModelRegistry } from '../../types'; import { ExtendedEntityFieldName, formatEntitiesForResponse } from '../../utils'; import { AugmentedEntityRecord } from '../../utils/formatEntity'; +import { isExtendedField } from '../../utils/extendedFieldFunctions'; const DEFAULT_FIELDS: ExtendedEntityFieldName[] = ['id', 'parent_name', 'code', 'name', 'type']; @@ -184,7 +185,7 @@ export const getEntityDescendants = async ({ [rootEntityId], { filter: dbEntityFilter, - fields, + fields: fields.filter(field => !isExtendedField(field)), pageSize, }, ); diff --git a/packages/datatrak-web/src/sync/ClientSyncManager.ts b/packages/datatrak-web/src/sync/ClientSyncManager.ts index 371a920e4f..07262f3a9d 100644 --- a/packages/datatrak-web/src/sync/ClientSyncManager.ts +++ b/packages/datatrak-web/src/sync/ClientSyncManager.ts @@ -424,7 +424,7 @@ export class ClientSyncManager { pullProgressCallback(records.length); }; - const batchSize = 200; + const batchSize = 10000; await pullIncomingChanges(this.models, sessionId, batchSize, processStreamedDataFunction); this.setProgress(this.progressMaxByStage[SYNC_STAGES.PERSIST - 1], 'Saving changes...'); diff --git a/packages/sync/src/utils/manageSnapshotTable.ts b/packages/sync/src/utils/manageSnapshotTable.ts index 9142099969..554ff44975 100644 --- a/packages/sync/src/utils/manageSnapshotTable.ts +++ b/packages/sync/src/utils/manageSnapshotTable.ts @@ -56,7 +56,7 @@ export const createClientSnapshotTable = async (database: TupaiaDatabase, sessio await database.executeSql(` CREATE TABLE ${tableName} ( id BIGSERIAL PRIMARY KEY, - record_type character varying(255) NOT NULL, + record_type TEXT NOT NULL, is_deleted BOOLEAN DEFAULT false, data json NOT NULL ) WITH ( From 26155783c5a9fe374a2abad74ff94de3570d394f Mon Sep 17 00:00:00 2001 From: chris-bes Date: Mon, 6 Oct 2025 11:31:10 +1100 Subject: [PATCH 23/26] fixed test --- .../central-server/src/tests/apiV2/entities/EditEntity.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/central-server/src/tests/apiV2/entities/EditEntity.test.js b/packages/central-server/src/tests/apiV2/entities/EditEntity.test.js index 552e88e743..0563a9e528 100644 --- a/packages/central-server/src/tests/apiV2/entities/EditEntity.test.js +++ b/packages/central-server/src/tests/apiV2/entities/EditEntity.test.js @@ -65,7 +65,7 @@ describe("Editing an entity's name", async () => { expect(result).to.deep.equal({ error: - "One of the following conditions need to be satisfied:\nNeed BES Admin access\nNeed Tupaia Admin Panel access to country 'SB' to edit entity", + "One of the following conditions need to be satisfied:\nNeed BES Admin access\nNeed Tupaia Admin Panel access to country ‘SB’ to edit entity ‘new_name’", }); }); }); From 9d83e78c3af96057c51d2f5708b2206a24ed9726 Mon Sep 17 00:00:00 2001 From: Jasper Lai <33956381+jaskfla@users.noreply.github.com> Date: Thu, 9 Oct 2025 10:31:02 +1300 Subject: [PATCH 24/26] dedupe import --- .../src/core/modelClasses/SurveyResponse/SurveyResponse.js | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/database/src/core/modelClasses/SurveyResponse/SurveyResponse.js b/packages/database/src/core/modelClasses/SurveyResponse/SurveyResponse.js index 11e27925c7..5c611478f3 100644 --- a/packages/database/src/core/modelClasses/SurveyResponse/SurveyResponse.js +++ b/packages/database/src/core/modelClasses/SurveyResponse/SurveyResponse.js @@ -1,4 +1,3 @@ -import log from 'winston'; import { difference, uniq } from 'es-toolkit'; import { flattenDeep, groupBy, keyBy } from 'es-toolkit/compat'; import log from 'winston'; From 2f714d9e6e254782aa56d6eaf3399397b60efe2f Mon Sep 17 00:00:00 2001 From: Jasper Lai <33956381+jaskfla@users.noreply.github.com> Date: Thu, 9 Oct 2025 11:11:54 +1300 Subject: [PATCH 25/26] dedupe another import --- .../datatrak-web/src/database/entity/getEntityDescendants.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/datatrak-web/src/database/entity/getEntityDescendants.ts b/packages/datatrak-web/src/database/entity/getEntityDescendants.ts index 3256e4914a..838bf4c303 100644 --- a/packages/datatrak-web/src/database/entity/getEntityDescendants.ts +++ b/packages/datatrak-web/src/database/entity/getEntityDescendants.ts @@ -14,7 +14,6 @@ import { DatatrakWebModelRegistry } from '../../types'; import { ExtendedEntityFieldName, formatEntitiesForResponse } from '../../utils'; import { isExtendedField } from '../../utils/extendedFieldFunctions'; import { AugmentedEntityRecord } from '../../utils/formatEntity'; -import { isExtendedField } from '../../utils/extendedFieldFunctions'; const DEFAULT_FIELDS: ExtendedEntityFieldName[] = ['id', 'parent_name', 'code', 'name', 'type']; From 759be741051865d4a79e688cb24619d6ebea18f0 Mon Sep 17 00:00:00 2001 From: chris-bes Date: Wed, 22 Oct 2025 09:46:11 +1100 Subject: [PATCH 26/26] addressed reviews --- packages/database/src/core/modelClasses/Entity.js | 2 +- packages/database/src/core/sync/buildSyncLookupSelect.js | 2 +- packages/datatrak-web/src/api/mutations/useEditUser.ts | 8 ++++---- packages/sync/src/utils/getDependencyOrder.ts | 3 ++- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/packages/database/src/core/modelClasses/Entity.js b/packages/database/src/core/modelClasses/Entity.js index 471843c010..853db7a9d5 100644 --- a/packages/database/src/core/modelClasses/Entity.js +++ b/packages/database/src/core/modelClasses/Entity.js @@ -337,7 +337,7 @@ export class EntityRecord extends DatabaseRecord { export class EntityModel extends MaterializedViewLogDatabaseModel { static syncDirection = SyncDirections.BIDIRECTIONAL; - get ExcludedFieldsFromSync() { + get excludedFieldsFromSync() { return ['point', 'bounds', 'region', 'parent_id']; } diff --git a/packages/database/src/core/sync/buildSyncLookupSelect.js b/packages/database/src/core/sync/buildSyncLookupSelect.js index 7e3a986d22..545e4a9938 100644 --- a/packages/database/src/core/sync/buildSyncLookupSelect.js +++ b/packages/database/src/core/sync/buildSyncLookupSelect.js @@ -4,7 +4,7 @@ export async function buildSyncLookupSelect(model, columns = {}) { const attributes = Object.keys(await model.fetchSchema()); const { projectIds } = columns; const table = model.databaseRecord; - const excludedFields = [...(model.ExcludedFieldsFromSync || []), ...COLUMNS_EXCLUDED_FROM_SYNC]; + const excludedFields = [...(model.excludedFieldsFromSync || []), ...COLUMNS_EXCLUDED_FROM_SYNC]; return ` SELECT diff --git a/packages/datatrak-web/src/api/mutations/useEditUser.ts b/packages/datatrak-web/src/api/mutations/useEditUser.ts index 5da082a243..de67dbc52f 100644 --- a/packages/datatrak-web/src/api/mutations/useEditUser.ts +++ b/packages/datatrak-web/src/api/mutations/useEditUser.ts @@ -38,12 +38,12 @@ export const useEditUser = (onSuccess?: () => void) => { await put('me', { data: updates }); }, { - onSuccess: async (_, variables) => { - await queryClient.invalidateQueries(['getUser']); + onSuccess: (_, variables) => { + queryClient.invalidateQueries(['getUser']); // If the user changes their project, we need to invalidate the entity descendants query so that recent entities are updated if they change back to the previous project without refreshing the page if (variables.projectId) { - await queryClient.invalidateQueries(['entityDescendants']); - await queryClient.invalidateQueries(['tasks']); + queryClient.invalidateQueries(['entityDescendants']); + queryClient.invalidateQueries(['tasks']); models.localSystemFact.addProjectForSync(variables.projectId); } diff --git a/packages/sync/src/utils/getDependencyOrder.ts b/packages/sync/src/utils/getDependencyOrder.ts index 5b2089c11b..679a4bfe38 100644 --- a/packages/sync/src/utils/getDependencyOrder.ts +++ b/packages/sync/src/utils/getDependencyOrder.ts @@ -1,6 +1,7 @@ import { compact, groupBy, mapValues } from 'lodash'; import { BaseDatabase, DatabaseModel } from '@tupaia/database'; +import { isNotNullish } from '@tupaia/tsutils'; interface Dependency { table_name: string; @@ -65,5 +66,5 @@ export const sortModelsByDependencyOrder = async ( return orderedDependencies .filter(dep => recordNames.has(dep)) .map(dep => models.find(r => r.databaseRecord === dep)) - .filter((model): model is DatabaseModel => model !== undefined); // Boolean does not work here. + .filter(isNotNullish); };