From 1324a47fe37d37925a0af1bff94c01282956c1a7 Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Wed, 16 Feb 2022 10:13:59 -0800 Subject: [PATCH 01/99] migrate app usage survey data to v3 --- .../mechanic/pods/MechanicUpgradePod.kt | 13 + .../upgrades/V3AppUsageSurveyMigration.kt | 333 ++++++++++++++++++ 2 files changed, 346 insertions(+) create mode 100644 src/main/kotlin/com/openlattice/mechanic/upgrades/V3AppUsageSurveyMigration.kt diff --git a/src/main/kotlin/com/openlattice/mechanic/pods/MechanicUpgradePod.kt b/src/main/kotlin/com/openlattice/mechanic/pods/MechanicUpgradePod.kt index b4301561..bfaa9232 100644 --- a/src/main/kotlin/com/openlattice/mechanic/pods/MechanicUpgradePod.kt +++ b/src/main/kotlin/com/openlattice/mechanic/pods/MechanicUpgradePod.kt @@ -55,12 +55,14 @@ import com.openlattice.ids.HazelcastIdGenerationService import com.openlattice.ids.HazelcastLongIdService import com.openlattice.ioc.providers.LateInitProvider import com.geekbeast.jdbc.DataSourceManager +import com.openlattice.data.DataGraphManager import com.openlattice.linking.LinkingQueryService import com.openlattice.linking.PostgresLinkingFeedbackService import com.openlattice.linking.graph.PostgresLinkingQueryService import com.openlattice.mechanic.MechanicCli.Companion.UPGRADE import com.openlattice.mechanic.Toolbox import com.openlattice.mechanic.upgrades.DeleteOrgMetadataEntitySets +import com.openlattice.mechanic.upgrades.V3AppUsageSurveyMigration import com.openlattice.mechanic.upgrades.V3StudyMigrationUpgrade import com.openlattice.organizations.roles.HazelcastPrincipalService import com.openlattice.organizations.roles.SecurePrincipalsManager @@ -365,6 +367,17 @@ class MechanicUpgradePod { ) } + @Bean fun v3AppUsageSurveyMigration(): V3AppUsageSurveyMigration { + return V3AppUsageSurveyMigration( + toolbox, + hikariDataSource, + principalsManager(), + searchService(), + dataQueryService(), + entitySetService() + ) + } + @PostConstruct fun post() { lateInitProvider.setDataSourceResolver(dataSourceResolver()) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/V3AppUsageSurveyMigration.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/V3AppUsageSurveyMigration.kt new file mode 100644 index 00000000..292fa948 --- /dev/null +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/V3AppUsageSurveyMigration.kt @@ -0,0 +1,333 @@ +package com.openlattice.mechanic.upgrades + +import com.geekbeast.postgres.PostgresArrays +import com.geekbeast.util.log +import com.openlattice.authorization.Principal +import com.openlattice.data.requests.NeighborEntityDetails +import com.openlattice.data.storage.MetadataOption +import com.openlattice.data.storage.postgres.PostgresEntityDataQueryService +import com.openlattice.datastore.services.EntitySetManager +import com.openlattice.edm.EdmConstants +import com.openlattice.edm.EdmConstants.Companion.LAST_WRITE_FQN +import com.openlattice.graph.PagedNeighborRequest +import com.openlattice.hazelcast.HazelcastMap +import com.openlattice.mechanic.Toolbox +import com.openlattice.organizations.roles.SecurePrincipalsManager +import com.openlattice.search.SearchService +import com.openlattice.search.requests.EntityNeighborsFilter +import com.zaxxer.hikari.HikariDataSource +import org.apache.olingo.commons.api.edm.FullQualifiedName +import org.slf4j.LoggerFactory +import java.time.OffsetDateTime +import java.util.* + +/** + * @author alfoncenzioka <alfonce@openlattice.com> + */ +class V3AppUsageSurveyMigration( + toolbox: Toolbox, + private val hds: HikariDataSource, + private val principalService: SecurePrincipalsManager, + private val searchService: SearchService, + private val dataQueryService: PostgresEntityDataQueryService, + private val entitySetService: EntitySetManager +) : Upgrade { + private val logger = LoggerFactory.getLogger(V3AppUsageSurveyMigration::class.java) + + private val organizations = HazelcastMap.ORGANIZATIONS.getMap(toolbox.hazelcast) + private val appConfigs = HazelcastMap.APP_CONFIGS.getMap(toolbox.hazelcast) + private val entitySetIds = HazelcastMap.ENTITY_SETS.getMap(toolbox.hazelcast).values.associateBy { it.name } + + private val allStudyIdsByParticipantEKID: MutableMap = mutableMapOf() + private val allParticipantIdsByEKID: MutableMap = mutableMapOf() + private val allSurveyDataByParticipantEKID: MutableMap>>> = mutableMapOf() + + companion object { + private val DATA_COLLECTION_APP_ID = UUID.fromString("c4e6d8fd-daf9-41e7-8c59-2a12c7ee0857") + + private val LEGACY_ORG_ID = UUID.fromString("7349c446-2acc-4d14-b2a9-a13be39cff93") + + private const val CHRONICLE_APP_ES_PREFIX = "chronicle_" + private const val DATA_COLLECTION_APP_PREFIX = "chronicle_data_collection_" + private const val USER_APPS = "userapps" + private const val USED_BY = "usedby" + private const val STUDIES = "studies" + private const val PARTICIPANTS = "participants" + private const val PARTICIPATED_IN = "participatedin" + + private const val LEGACY_USER_APPS_ES = "chronicle_user_apps" + private const val LEGACY_STUDIES_ES = "chronicle_study" + private const val LEGACY_PARTICIPATED_IN_ES = "chronicle_participated_in" + private const val LEGACY_USED_BY_ES = "chronicle_used_by" + + private val OL_ID_FQN = EdmConstants.ID_FQN + private val STRING_ID_FQN = FullQualifiedName("general.stringid") + private val PERSON_FQN = FullQualifiedName("nc.SubjectIdentification") + private val USER_FQN = FullQualifiedName("ol.user") + private val FULL_NAME_FQN = FullQualifiedName("general.fullname") + private val DATETIME_FQN = FullQualifiedName("ol.datetime") + private val TIMEZONE_FQN = FullQualifiedName("ol.timezone") + private val TITLE_FQN = FullQualifiedName("ol.title") + + private val FQN_TO_COLUMNS = mapOf( + LAST_WRITE_FQN to "submission_date", + TITLE_FQN to "application_label", + FULL_NAME_FQN to "app_package_name", + DATETIME_FQN to "event_timestamp", + TIMEZONE_FQN to "timezone", + USER_FQN to "users" + ) + + private val column_names = listOf("study_id", "participant_id") + FQN_TO_COLUMNS.values.toList() + private val APP_USAGE_SURVEY_COLUMNS = column_names.joinToString(",") { it } + private val APP_USAGE_SURVEY_PARAMS = column_names.joinToString(",") { "?" } + + /** + * PreparedStatement bind order + * 1) studyId + * 2) participantId + * 3) submissionDate, + * 4) appLabel + * 5) packageName + * 6) timestamp + * 7) timezone + * 8) users + */ + private val INSERT_INTO_APP_USAGE_SQL = """ + INSERT INTO app_usage_survey($APP_USAGE_SURVEY_COLUMNS) values ($APP_USAGE_SURVEY_PARAMS) + ON CONFLICT DO NOTHING + """.trimIndent() + + } + + override fun upgrade(): Boolean { + getAllAppUsageSurveyData() + + logger.info("migrating app usage surveys to v3") + + try { + val written = hds.connection.use { connection -> + connection.prepareStatement(INSERT_INTO_APP_USAGE_SQL).use { ps -> + allSurveyDataByParticipantEKID.forEach { (key, data) -> + val studyId = allStudyIdsByParticipantEKID[key] + val participantId = allParticipantIdsByEKID[key] + + if (studyId == null || participantId == null) { + logger.warn("Skipping migration for participant $key. failed to retrieve associated studyId or participantId") + return@forEach + } + + ps.setObject(1, studyId) + ps.setString(2, participantId) + + data.forEach data@{ entity -> + val users = entity.getOrDefault(USER_FQN, listOf()) + .map { it.toString() }.filter { it.isNotBlank() }.toList() + + if (users.isEmpty() || users.first().toString().isBlank()) { + return@data + } + + val submissionDate = getFirstValueOrNull(entity, LAST_WRITE_FQN) + val applicationLabel = getFirstValueOrNull(entity, TITLE_FQN) + val appPackageName = getFirstValueOrNull(entity, FULL_NAME_FQN) + val timezone = getFirstValueOrNull(entity, TIMEZONE_FQN) + val timestamp = getFirstValueOrNull(entity, DATETIME_FQN) + + if (submissionDate == null || appPackageName == null || timestamp == null) { + return@data + } + + ps.setObject(3, OffsetDateTime.parse(submissionDate)) + ps.setString(4, applicationLabel) + ps.setString(5, appPackageName) + ps.setObject(6, OffsetDateTime.parse(timestamp)) + ps.setString(7, timezone) + ps.setArray(8, PostgresArrays.createTextArray(connection, users)) + ps.addBatch() + } + } + ps.executeBatch().sum() + } + } + + logger.info("wrote {} entities to db. data query service returned {} entities", written, allSurveyDataByParticipantEKID.values.flatten().size) + return true + } catch (ex: Exception) { + logger.error("error migrating app usage survey data to v3", ex) + return false + } + } + + private fun getAllAppUsageSurveyData() { + val v2DataCollectionOrgIds = appConfigs.keys + .filter { it.appId == DATA_COLLECTION_APP_ID } + .map { it.organizationId } + .toMutableSet() + + (v2DataCollectionOrgIds + LEGACY_ORG_ID).forEach org@{ orgId -> + logger.info("======================================") + logger.info("getting app usage survey data for org $orgId") + logger.info("=====================================") + + // entity sets + val orgEntitySetIds = getOrgEntitySetIds(orgId) + + val studiesEntitySetId = orgEntitySetIds.getValue(STUDIES) + val participatedInEntitySetId = orgEntitySetIds.getValue(PARTICIPATED_IN) + val participantsEntitySetId = orgEntitySetIds[PARTICIPANTS] // this will be null if org is legacy + val userAppsEntitySetId = orgEntitySetIds.getValue(USER_APPS) + val usedByEntitySetId = orgEntitySetIds.getValue(USED_BY) + + val adminRoleAclKey = organizations[orgId]?.adminRoleAclKey + if (adminRoleAclKey == null) { + logger.warn("skipping {} since it doesn't have admin role", orgId) + return@org + } + val adminPrincipals = principalService.getAllUsersWithPrincipal(adminRoleAclKey).map { it.principal }.toSet() + + val studies = filter(getEntitiesByEntityKeyId(studiesEntitySetId)) + if (studies.isEmpty()) { + logger.info("org {} has no studies. skipping", orgId) + return@org + } + logger.info("found {} studies in org {}", studies.size, orgId) + + val studyEntityKeyIds = studies.keys + val studyIds = studies.map { getFirstUUIDOrNull(it.value, STRING_ID_FQN) }.filterNotNull().toSet() + + val participantEntitySetIds = when (orgId) { + LEGACY_ORG_ID -> getLegacyParticipantEntitySetIds(studyIds) + else -> setOf(participantsEntitySetId!!) + } + val participants = getStudyParticipants(studiesEntitySetId, studyEntityKeyIds, participantEntitySetIds, participatedInEntitySetId, adminPrincipals) + + val participantsByStudyId = participants.mapValues { (_, neighbor) -> neighbor.map { it.neighborId.get() }.toSet() } + logger.info("org {} participant count by study {}", orgId, participantsByStudyId.mapValues { it.value.size }) + + val participantIdByEKID = + participants.values + .flatten() + .associate { getFirstUUIDOrNull(it.neighborDetails.get(), OL_ID_FQN)!! to getFirstValueOrNull(it.neighborDetails.get(), PERSON_FQN) } + val studyIdByParticipantEKID = participantsByStudyId + .map { (studyId, participants) -> participants.associateWith { studyId } } + .flatMap { + it.asSequence() + }.associate { it.key to it.value } + + val surveysDataByParticipant = getSurveysDataByParticipant( + adminPrincipals, participantsByStudyId.values.flatten().toSet(), participantEntitySetIds, usedByEntitySetId, userAppsEntitySetId) + + logger.info("found {} survey entities in org {}", surveysDataByParticipant.values.size, orgId) + + allStudyIdsByParticipantEKID += studyIdByParticipantEKID + allParticipantIdsByEKID += participantIdByEKID + allSurveyDataByParticipantEKID += surveysDataByParticipant + } + } + + private fun filter(entities: Map>>): Map>> { + return entities.filterValues { getFirstUUIDOrNull(it, STRING_ID_FQN) != null } + } + + // returns a map of collection template name to entity set id + private fun getOrgEntitySetIds(organizationId: UUID): Map { + val entitySetNameByTemplateName = when (organizationId) { + LEGACY_ORG_ID -> mapOf( + USER_APPS to LEGACY_USER_APPS_ES, + USED_BY to LEGACY_USED_BY_ES, + STUDIES to LEGACY_STUDIES_ES, + PARTICIPATED_IN to LEGACY_PARTICIPATED_IN_ES + ) + else -> { + val orgIdToStr = organizationId.toString().replace("-", "") + mapOf( + USER_APPS to "$DATA_COLLECTION_APP_PREFIX${orgIdToStr}_$USER_APPS", + USED_BY to "$DATA_COLLECTION_APP_PREFIX${orgIdToStr}_$USED_BY", + STUDIES to "$CHRONICLE_APP_ES_PREFIX${orgIdToStr}_$STUDIES", + PARTICIPATED_IN to "$CHRONICLE_APP_ES_PREFIX${orgIdToStr}_$PARTICIPATED_IN", + PARTICIPANTS to "$CHRONICLE_APP_ES_PREFIX${orgIdToStr}_${PARTICIPANTS}" + ) + } + } + + return entitySetNameByTemplateName.mapValues { entitySetIds.getValue(it.value).id } + } + + private fun getLegacyParticipantEntitySetIds(studyIds: Set): Set { + val entitySetNames = studyIds.map { "chronicle_participants_$it" } + return entitySetNames.map { entitySetIds.getValue(it).id }.toSet() + } + + private fun getStudyParticipants( + studiesEntitySetId: UUID, + studyEntityKeyIds: Set, + participantEntitySetIds: Set, + participatedInEntitySetId: UUID, + principals: Set + + ): Map> { + val filter = EntityNeighborsFilter( + studyEntityKeyIds, Optional.of(participantEntitySetIds), Optional.of(setOf(studiesEntitySetId)), Optional.of(setOf(participatedInEntitySetId))) + return searchService.executeEntityNeighborSearch(setOf(studiesEntitySetId), PagedNeighborRequest(filter), principals).neighbors + } + + private fun getSurveysDataByParticipant( + principals: Set, + entityKeyIds: Set, + participantEntitySetIds: Set, + usedByEnEntitySetId: UUID, + userAppsEntitySetId: UUID): Map>>> { + + val filter = EntityNeighborsFilter(entityKeyIds, Optional.of(setOf(userAppsEntitySetId)), Optional.of(participantEntitySetIds), Optional.of(setOf(usedByEnEntitySetId))) + + val neighbors = searchService + .executeEntityNeighborSearch(participantEntitySetIds, PagedNeighborRequest(filter), principals).neighbors + .mapValues { (_, v) -> + v.associateBy { getFirstUUIDOrNull(it.associationDetails, OL_ID_FQN)!! } + } + + val associationEntityKeyIds = neighbors.values.flatMap { it.keys }.toSet() + val usedByEntities = getEntitiesByEntityKeyId(usedByEnEntitySetId, associationEntityKeyIds, setOf(MetadataOption.LAST_WRITE)) + + return neighbors + .mapValues { (_, v) -> + v.map { + it.value.neighborDetails.get().toMutableMap() + usedByEntities.getOrDefault(it.key, mapOf()) + } + } + } + + private fun getFirstUUIDOrNull(entity: Map>, fqn: FullQualifiedName): UUID? { + return when (val string = getFirstValueOrNull(entity, fqn)) { + null -> null + else -> UUID.fromString(string) + } + } + + private fun getFirstValueOrNull(entity: Map>, fqn: FullQualifiedName): String? { + entity[fqn]?.iterator()?.let { + if (it.hasNext()) return it.next().toString() + } + return null + } + + private fun getEntitiesByEntityKeyId( + entitySetId: UUID, + entityKeyIds: Set = setOf(), + metadataOptions: Set = setOf() + ): Map>> { + return dataQueryService.getEntitiesWithPropertyTypeFqns( + mapOf(entitySetId to Optional.of(entityKeyIds)), + entitySetService.getPropertyTypesOfEntitySets(setOf(entitySetId)), + mapOf(), + metadataOptions, + Optional.empty(), + false, + ) + } + + override fun getSupportedVersion(): Long { + return Version.V2021_07_23.value + } +} From 1b51a550e56a5aa6b88695924dae1a60cdff65d4 Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Wed, 16 Feb 2022 10:15:43 -0800 Subject: [PATCH 02/99] remove unused imports --- .../kotlin/com/openlattice/mechanic/pods/MechanicUpgradePod.kt | 1 - .../openlattice/mechanic/upgrades/V3AppUsageSurveyMigration.kt | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/pods/MechanicUpgradePod.kt b/src/main/kotlin/com/openlattice/mechanic/pods/MechanicUpgradePod.kt index bfaa9232..f7d42d6e 100644 --- a/src/main/kotlin/com/openlattice/mechanic/pods/MechanicUpgradePod.kt +++ b/src/main/kotlin/com/openlattice/mechanic/pods/MechanicUpgradePod.kt @@ -55,7 +55,6 @@ import com.openlattice.ids.HazelcastIdGenerationService import com.openlattice.ids.HazelcastLongIdService import com.openlattice.ioc.providers.LateInitProvider import com.geekbeast.jdbc.DataSourceManager -import com.openlattice.data.DataGraphManager import com.openlattice.linking.LinkingQueryService import com.openlattice.linking.PostgresLinkingFeedbackService import com.openlattice.linking.graph.PostgresLinkingQueryService diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/V3AppUsageSurveyMigration.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/V3AppUsageSurveyMigration.kt index 292fa948..f67f353b 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/V3AppUsageSurveyMigration.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/V3AppUsageSurveyMigration.kt @@ -1,7 +1,6 @@ package com.openlattice.mechanic.upgrades import com.geekbeast.postgres.PostgresArrays -import com.geekbeast.util.log import com.openlattice.authorization.Principal import com.openlattice.data.requests.NeighborEntityDetails import com.openlattice.data.storage.MetadataOption @@ -121,7 +120,7 @@ class V3AppUsageSurveyMigration( ps.setString(2, participantId) data.forEach data@{ entity -> - val users = entity.getOrDefault(USER_FQN, listOf()) + val users = entity.getOrDefault(USER_FQN, listOf()) .map { it.toString() }.filter { it.isNotBlank() }.toList() if (users.isEmpty() || users.first().toString().isBlank()) { From 0357f19475df0af010f16ca6b1f864b234a6faf8 Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Wed, 16 Feb 2022 10:20:23 -0800 Subject: [PATCH 03/99] rename --- .../mechanic/upgrades/V3AppUsageSurveyMigration.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/V3AppUsageSurveyMigration.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/V3AppUsageSurveyMigration.kt index f67f353b..2ab59ffe 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/V3AppUsageSurveyMigration.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/V3AppUsageSurveyMigration.kt @@ -185,7 +185,7 @@ class V3AppUsageSurveyMigration( } val adminPrincipals = principalService.getAllUsersWithPrincipal(adminRoleAclKey).map { it.principal }.toSet() - val studies = filter(getEntitiesByEntityKeyId(studiesEntitySetId)) + val studies = filterInvalidStudies(getEntitiesByEntityKeyId(studiesEntitySetId)) if (studies.isEmpty()) { logger.info("org {} has no studies. skipping", orgId) return@org @@ -225,7 +225,7 @@ class V3AppUsageSurveyMigration( } } - private fun filter(entities: Map>>): Map>> { + private fun filterInvalidStudies(entities: Map>>): Map>> { return entities.filterValues { getFirstUUIDOrNull(it, STRING_ID_FQN) != null } } From 6c99baa14361aafa3827936573a302b80403180d Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Wed, 16 Feb 2022 15:24:24 -0800 Subject: [PATCH 04/99] use chronicle super user principals for data query --- .../mechanic/pods/MechanicUpgradePod.kt | 2 +- .../upgrades/V3AppUsageSurveyMigration.kt | 28 +++++++++++++++---- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/pods/MechanicUpgradePod.kt b/src/main/kotlin/com/openlattice/mechanic/pods/MechanicUpgradePod.kt index f7d42d6e..dae2736b 100644 --- a/src/main/kotlin/com/openlattice/mechanic/pods/MechanicUpgradePod.kt +++ b/src/main/kotlin/com/openlattice/mechanic/pods/MechanicUpgradePod.kt @@ -369,7 +369,7 @@ class MechanicUpgradePod { @Bean fun v3AppUsageSurveyMigration(): V3AppUsageSurveyMigration { return V3AppUsageSurveyMigration( toolbox, - hikariDataSource, + rhizomeConfiguration, principalsManager(), searchService(), dataQueryService(), diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/V3AppUsageSurveyMigration.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/V3AppUsageSurveyMigration.kt index 2ab59ffe..c613e63f 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/V3AppUsageSurveyMigration.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/V3AppUsageSurveyMigration.kt @@ -1,6 +1,7 @@ package com.openlattice.mechanic.upgrades import com.geekbeast.postgres.PostgresArrays +import com.geekbeast.rhizome.configuration.RhizomeConfiguration import com.openlattice.authorization.Principal import com.openlattice.data.requests.NeighborEntityDetails import com.openlattice.data.storage.MetadataOption @@ -14,6 +15,7 @@ import com.openlattice.mechanic.Toolbox import com.openlattice.organizations.roles.SecurePrincipalsManager import com.openlattice.search.SearchService import com.openlattice.search.requests.EntityNeighborsFilter +import com.zaxxer.hikari.HikariConfig import com.zaxxer.hikari.HikariDataSource import org.apache.olingo.commons.api.edm.FullQualifiedName import org.slf4j.LoggerFactory @@ -25,7 +27,7 @@ import java.util.* */ class V3AppUsageSurveyMigration( toolbox: Toolbox, - private val hds: HikariDataSource, + private val rhizomeConfiguration: RhizomeConfiguration, private val principalService: SecurePrincipalsManager, private val searchService: SearchService, private val dataQueryService: PostgresEntityDataQueryService, @@ -41,6 +43,9 @@ class V3AppUsageSurveyMigration( private val allParticipantIdsByEKID: MutableMap = mutableMapOf() private val allSurveyDataByParticipantEKID: MutableMap>>> = mutableMapOf() + //TODO: replace empty strings with chronicle super user ids (auth0 and google-oauth2) when running migration + private val chronicleSuperUserIds = setOf("", "") + companion object { private val DATA_COLLECTION_APP_ID = UUID.fromString("c4e6d8fd-daf9-41e7-8c59-2a12c7ee0857") @@ -104,6 +109,10 @@ class V3AppUsageSurveyMigration( logger.info("migrating app usage surveys to v3") + val (hikariConfiguration) = rhizomeConfiguration.datasourceConfigurations["chronicle"]!! + val hc = HikariConfig(hikariConfiguration) + val hds = HikariDataSource(hc) + try { val written = hds.connection.use { connection -> connection.prepareStatement(INSERT_INTO_APP_USAGE_SQL).use { ps -> @@ -164,6 +173,8 @@ class V3AppUsageSurveyMigration( .map { it.organizationId } .toMutableSet() + val superUserPrincipals = getChronicleSuperUserPrincipals() + (v2DataCollectionOrgIds + LEGACY_ORG_ID).forEach org@{ orgId -> logger.info("======================================") logger.info("getting app usage survey data for org $orgId") @@ -183,7 +194,7 @@ class V3AppUsageSurveyMigration( logger.warn("skipping {} since it doesn't have admin role", orgId) return@org } - val adminPrincipals = principalService.getAllUsersWithPrincipal(adminRoleAclKey).map { it.principal }.toSet() + val principals = principalService.getAllUsersWithPrincipal(adminRoleAclKey).map { it.principal }.toSet() + superUserPrincipals val studies = filterInvalidStudies(getEntitiesByEntityKeyId(studiesEntitySetId)) if (studies.isEmpty()) { @@ -199,7 +210,7 @@ class V3AppUsageSurveyMigration( LEGACY_ORG_ID -> getLegacyParticipantEntitySetIds(studyIds) else -> setOf(participantsEntitySetId!!) } - val participants = getStudyParticipants(studiesEntitySetId, studyEntityKeyIds, participantEntitySetIds, participatedInEntitySetId, adminPrincipals) + val participants = getStudyParticipants(studiesEntitySetId, studyEntityKeyIds, participantEntitySetIds, participatedInEntitySetId, principals) val participantsByStudyId = participants.mapValues { (_, neighbor) -> neighbor.map { it.neighborId.get() }.toSet() } logger.info("org {} participant count by study {}", orgId, participantsByStudyId.mapValues { it.value.size }) @@ -215,9 +226,9 @@ class V3AppUsageSurveyMigration( }.associate { it.key to it.value } val surveysDataByParticipant = getSurveysDataByParticipant( - adminPrincipals, participantsByStudyId.values.flatten().toSet(), participantEntitySetIds, usedByEntitySetId, userAppsEntitySetId) + principals, participantsByStudyId.values.flatten().toSet(), participantEntitySetIds, usedByEntitySetId, userAppsEntitySetId) - logger.info("found {} survey entities in org {}", surveysDataByParticipant.values.size, orgId) + logger.info("found {} survey entities in org {}", surveysDataByParticipant.values.flatten().size, orgId) allStudyIdsByParticipantEKID += studyIdByParticipantEKID allParticipantIdsByEKID += participantIdByEKID @@ -225,6 +236,13 @@ class V3AppUsageSurveyMigration( } } + private fun getChronicleSuperUserPrincipals(): Set { + return chronicleSuperUserIds + .map { principalService.getSecurablePrincipal(it)} + .map { principalService.getAllPrincipals(it).map { principal -> principal.principal } } + .flatten().toSet() + } + private fun filterInvalidStudies(entities: Map>>): Map>> { return entities.filterValues { getFirstUUIDOrNull(it, STRING_ID_FQN) != null } } From d458003cb457f56f4d3aa432d726bec1a0d3783f Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Mon, 21 Feb 2022 06:56:48 -0800 Subject: [PATCH 05/99] create table if not exists --- .../upgrades/V3AppUsageSurveyMigration.kt | 53 +++++++++++++++---- 1 file changed, 43 insertions(+), 10 deletions(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/V3AppUsageSurveyMigration.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/V3AppUsageSurveyMigration.kt index c613e63f..9781d9fb 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/V3AppUsageSurveyMigration.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/V3AppUsageSurveyMigration.kt @@ -44,7 +44,7 @@ class V3AppUsageSurveyMigration( private val allSurveyDataByParticipantEKID: MutableMap>>> = mutableMapOf() //TODO: replace empty strings with chronicle super user ids (auth0 and google-oauth2) when running migration - private val chronicleSuperUserIds = setOf("", "") + private val chronicleSuperUserIds = setOf("google-oauth2|113860246540203337319", "auth0|5ae9026c04eb0b243f1d2bb6") companion object { private val DATA_COLLECTION_APP_ID = UUID.fromString("c4e6d8fd-daf9-41e7-8c59-2a12c7ee0857") @@ -86,6 +86,8 @@ class V3AppUsageSurveyMigration( private val APP_USAGE_SURVEY_COLUMNS = column_names.joinToString(",") { it } private val APP_USAGE_SURVEY_PARAMS = column_names.joinToString(",") { "?" } + private const val TABLE_NAME = "public.app_usage_survey" + /** * PreparedStatement bind order * 1) studyId @@ -102,6 +104,30 @@ class V3AppUsageSurveyMigration( ON CONFLICT DO NOTHING """.trimIndent() + private val CREATE_APP_USAGE_SURVEY_TABLE_SQL = """ + CREATE TABLE IF NOT EXISTS $TABLE_NAME( + study_id uuid NOT NULL, + participant_id text NOT NULL, + submission_date timestamp with time zone NOT NULL, + application_label text, + app_package_name text NOT NULL, + event_timestamp timestamp with time zone NOT NULL, + timezone text, + users text[], + PRIMARY KEY(app_package_name, event_timestamp) + ); + """.trimIndent() + } + + init { + // TODO: change chronicle to alpr + val (hikariConfiguration) = rhizomeConfiguration.datasourceConfigurations["chronicle"]!! + val hc = HikariConfig(hikariConfiguration) + val hds = HikariDataSource(hc) + + hds.connection.createStatement().use { stmt -> + stmt.execute(CREATE_APP_USAGE_SURVEY_TABLE_SQL) + } } override fun upgrade(): Boolean { @@ -109,6 +135,7 @@ class V3AppUsageSurveyMigration( logger.info("migrating app usage surveys to v3") + // TODO: change chronicle to alpr val (hikariConfiguration) = rhizomeConfiguration.datasourceConfigurations["chronicle"]!! val hc = HikariConfig(hikariConfiguration) val hds = HikariDataSource(hc) @@ -140,19 +167,21 @@ class V3AppUsageSurveyMigration( val applicationLabel = getFirstValueOrNull(entity, TITLE_FQN) val appPackageName = getFirstValueOrNull(entity, FULL_NAME_FQN) val timezone = getFirstValueOrNull(entity, TIMEZONE_FQN) - val timestamp = getFirstValueOrNull(entity, DATETIME_FQN) - if (submissionDate == null || appPackageName == null || timestamp == null) { + if (submissionDate == null || appPackageName == null) { return@data } - ps.setObject(3, OffsetDateTime.parse(submissionDate)) - ps.setString(4, applicationLabel) - ps.setString(5, appPackageName) - ps.setObject(6, OffsetDateTime.parse(timestamp)) - ps.setString(7, timezone) - ps.setArray(8, PostgresArrays.createTextArray(connection, users)) - ps.addBatch() + val timestamps = getAppUsageTimestamps(entity) + timestamps.forEach { timestamp -> + ps.setObject(3, OffsetDateTime.parse(submissionDate)) + ps.setString(4, applicationLabel) + ps.setString(5, appPackageName) + ps.setObject(6, OffsetDateTime.parse(timestamp)) + ps.setString(7, timezone) + ps.setArray(8, PostgresArrays.createTextArray(connection, users)) + ps.addBatch() + } } } ps.executeBatch().sum() @@ -315,6 +344,10 @@ class V3AppUsageSurveyMigration( } } + private fun getAppUsageTimestamps(entity: Map>): Set { + return entity[DATETIME_FQN]?.map { it.toString() }?.toSet() ?: setOf() + } + private fun getFirstUUIDOrNull(entity: Map>, fqn: FullQualifiedName): UUID? { return when (val string = getFirstValueOrNull(entity, fqn)) { null -> null From d451961e6c001903c75e30c2f97f5daa7c4e0882 Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Mon, 21 Feb 2022 09:07:04 -0800 Subject: [PATCH 06/99] remove user ids --- .../mechanic/upgrades/V3AppUsageSurveyMigration.kt | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/V3AppUsageSurveyMigration.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/V3AppUsageSurveyMigration.kt index 9781d9fb..928d1a08 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/V3AppUsageSurveyMigration.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/V3AppUsageSurveyMigration.kt @@ -44,7 +44,7 @@ class V3AppUsageSurveyMigration( private val allSurveyDataByParticipantEKID: MutableMap>>> = mutableMapOf() //TODO: replace empty strings with chronicle super user ids (auth0 and google-oauth2) when running migration - private val chronicleSuperUserIds = setOf("google-oauth2|113860246540203337319", "auth0|5ae9026c04eb0b243f1d2bb6") + private val chronicleSuperUserIds = setOf("", "") companion object { private val DATA_COLLECTION_APP_ID = UUID.fromString("c4e6d8fd-daf9-41e7-8c59-2a12c7ee0857") @@ -172,7 +172,7 @@ class V3AppUsageSurveyMigration( return@data } - val timestamps = getAppUsageTimestamps(entity) + val timestamps = entity[DATETIME_FQN]?.map { it.toString() }?.toSet() ?: setOf() timestamps.forEach { timestamp -> ps.setObject(3, OffsetDateTime.parse(submissionDate)) ps.setString(4, applicationLabel) @@ -344,10 +344,6 @@ class V3AppUsageSurveyMigration( } } - private fun getAppUsageTimestamps(entity: Map>): Set { - return entity[DATETIME_FQN]?.map { it.toString() }?.toSet() ?: setOf() - } - private fun getFirstUUIDOrNull(entity: Map>, fqn: FullQualifiedName): UUID? { return when (val string = getFirstValueOrNull(entity, fqn)) { null -> null From 83637c52f5bb0ac61b9f929ee5ee3f344e0d481c Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Mon, 21 Feb 2022 11:42:13 -0800 Subject: [PATCH 07/99] fix gradle/aws --- build.gradle | 4 ++-- src/main/resources/aws.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/build.gradle b/build.gradle index 32df8d1d..4eba2449 100644 --- a/build.gradle +++ b/build.gradle @@ -50,11 +50,11 @@ def PARALLEL = "$System.env.PARALLELISM" def GC = "$System.env.GC" if (MECHANIC_XMS == 'null' || MECHANIC_XMS == null || MECHANIC_XMS == "") { - MECHANIC_XMS = '-Xms1g' + MECHANIC_XMS = '-Xms100g' } if (MECHANIC_XMX == 'null' || MECHANIC_XMX == null || MECHANIC_XMX == "") { - MECHANIC_XMX = '-Xmx4g' + MECHANIC_XMX = '-Xmx200g' } if (MECHANIC_ARGS == 'null' || MECHANIC_ARGS == null || MECHANIC_ARGS == "") { diff --git a/src/main/resources/aws.yaml b/src/main/resources/aws.yaml index 093870d2..1346b840 100644 --- a/src/main/resources/aws.yaml +++ b/src/main/resources/aws.yaml @@ -1,3 +1,3 @@ -bucket: "lattice-prod-config" +bucket: "openlattice-alpha-config" folder: "mechanic" region: us-gov-west-1 \ No newline at end of file From d4eecf0e582d3c59e4e775f0444e286758812b2e Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Mon, 21 Feb 2022 11:44:31 -0800 Subject: [PATCH 08/99] super user ids --- .../mechanic/upgrades/V3AppUsageSurveyMigration.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/V3AppUsageSurveyMigration.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/V3AppUsageSurveyMigration.kt index 928d1a08..2ceb5299 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/V3AppUsageSurveyMigration.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/V3AppUsageSurveyMigration.kt @@ -44,7 +44,7 @@ class V3AppUsageSurveyMigration( private val allSurveyDataByParticipantEKID: MutableMap>>> = mutableMapOf() //TODO: replace empty strings with chronicle super user ids (auth0 and google-oauth2) when running migration - private val chronicleSuperUserIds = setOf("", "") + private val chronicleSuperUserIds = setOf(" auth0|5ae9026c04eb0b243f1d2bb6", "google-oauth2|113860246540203337319") companion object { private val DATA_COLLECTION_APP_ID = UUID.fromString("c4e6d8fd-daf9-41e7-8c59-2a12c7ee0857") @@ -121,7 +121,7 @@ class V3AppUsageSurveyMigration( init { // TODO: change chronicle to alpr - val (hikariConfiguration) = rhizomeConfiguration.datasourceConfigurations["chronicle"]!! + val (hikariConfiguration) = rhizomeConfiguration.datasourceConfigurations["alpr"]!! val hc = HikariConfig(hikariConfiguration) val hds = HikariDataSource(hc) @@ -136,7 +136,7 @@ class V3AppUsageSurveyMigration( logger.info("migrating app usage surveys to v3") // TODO: change chronicle to alpr - val (hikariConfiguration) = rhizomeConfiguration.datasourceConfigurations["chronicle"]!! + val (hikariConfiguration) = rhizomeConfiguration.datasourceConfigurations["alpr"]!! val hc = HikariConfig(hikariConfiguration) val hds = HikariDataSource(hc) From 8ff714db3d92189a5136ab0099c8a091270892c4 Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Mon, 21 Feb 2022 12:13:12 -0800 Subject: [PATCH 09/99] use principalService --- .../kotlin/com/openlattice/mechanic/pods/MechanicUpgradePod.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/pods/MechanicUpgradePod.kt b/src/main/kotlin/com/openlattice/mechanic/pods/MechanicUpgradePod.kt index 24efca5c..231a6ade 100644 --- a/src/main/kotlin/com/openlattice/mechanic/pods/MechanicUpgradePod.kt +++ b/src/main/kotlin/com/openlattice/mechanic/pods/MechanicUpgradePod.kt @@ -384,7 +384,7 @@ class MechanicUpgradePod { return V3AppUsageSurveyMigration( toolbox, rhizomeConfiguration, - principalsManager(), + principalService(), searchService(), dataQueryService(), entitySetService() From 720ccbf7f8ad4137f9058094eb0194563134c9e2 Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Mon, 21 Feb 2022 12:50:47 -0800 Subject: [PATCH 10/99] fix user id --- .../openlattice/mechanic/upgrades/V3AppUsageSurveyMigration.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/V3AppUsageSurveyMigration.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/V3AppUsageSurveyMigration.kt index 2ceb5299..be04c2b9 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/V3AppUsageSurveyMigration.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/V3AppUsageSurveyMigration.kt @@ -44,7 +44,7 @@ class V3AppUsageSurveyMigration( private val allSurveyDataByParticipantEKID: MutableMap>>> = mutableMapOf() //TODO: replace empty strings with chronicle super user ids (auth0 and google-oauth2) when running migration - private val chronicleSuperUserIds = setOf(" auth0|5ae9026c04eb0b243f1d2bb6", "google-oauth2|113860246540203337319") + private val chronicleSuperUserIds = setOf("auth0|5ae9026c04eb0b243f1d2bb6", "google-oauth2|113860246540203337319") companion object { private val DATA_COLLECTION_APP_ID = UUID.fromString("c4e6d8fd-daf9-41e7-8c59-2a12c7ee0857") From 687cd35550e1ae20be04f7b5ce198f4888bee178 Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Mon, 21 Feb 2022 14:28:58 -0800 Subject: [PATCH 11/99] fix gradle --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 4eba2449..d0016a34 100644 --- a/build.gradle +++ b/build.gradle @@ -54,7 +54,7 @@ if (MECHANIC_XMS == 'null' || MECHANIC_XMS == null || MECHANIC_XMS == "") { } if (MECHANIC_XMX == 'null' || MECHANIC_XMX == null || MECHANIC_XMX == "") { - MECHANIC_XMX = '-Xmx200g' + MECHANIC_XMX = '-Xmx100g' } if (MECHANIC_ARGS == 'null' || MECHANIC_ARGS == null || MECHANIC_ARGS == "") { From 59bd08d9a838f9286deae8d36c5fd2636d2485ec Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Mon, 21 Feb 2022 14:50:51 -0800 Subject: [PATCH 12/99] undo --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index d0016a34..4eba2449 100644 --- a/build.gradle +++ b/build.gradle @@ -54,7 +54,7 @@ if (MECHANIC_XMS == 'null' || MECHANIC_XMS == null || MECHANIC_XMS == "") { } if (MECHANIC_XMX == 'null' || MECHANIC_XMX == null || MECHANIC_XMX == "") { - MECHANIC_XMX = '-Xmx100g' + MECHANIC_XMX = '-Xmx200g' } if (MECHANIC_ARGS == 'null' || MECHANIC_ARGS == null || MECHANIC_ARGS == "") { From 27f5ccab59004254cac56e8856da28f24f6dcd79 Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Mon, 21 Feb 2022 15:26:01 -0800 Subject: [PATCH 13/99] filter out nulls --- .../openlattice/mechanic/upgrades/V3AppUsageSurveyMigration.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/V3AppUsageSurveyMigration.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/V3AppUsageSurveyMigration.kt index be04c2b9..13d2e1c8 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/V3AppUsageSurveyMigration.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/V3AppUsageSurveyMigration.kt @@ -302,7 +302,7 @@ class V3AppUsageSurveyMigration( private fun getLegacyParticipantEntitySetIds(studyIds: Set): Set { val entitySetNames = studyIds.map { "chronicle_participants_$it" } - return entitySetNames.map { entitySetIds.getValue(it).id }.toSet() + return entitySetNames.mapNotNull { entitySetIds[it]?.id }.toSet() } private fun getStudyParticipants( From e90c741cd4b963e18dec97bd5b97d99c7fbee779 Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Mon, 21 Feb 2022 15:57:21 -0800 Subject: [PATCH 14/99] check authorization --- .../com/openlattice/mechanic/pods/MechanicUpgradePod.kt | 1 + .../mechanic/upgrades/V3AppUsageSurveyMigration.kt | 8 ++++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/pods/MechanicUpgradePod.kt b/src/main/kotlin/com/openlattice/mechanic/pods/MechanicUpgradePod.kt index 231a6ade..5a2fbb37 100644 --- a/src/main/kotlin/com/openlattice/mechanic/pods/MechanicUpgradePod.kt +++ b/src/main/kotlin/com/openlattice/mechanic/pods/MechanicUpgradePod.kt @@ -384,6 +384,7 @@ class MechanicUpgradePod { return V3AppUsageSurveyMigration( toolbox, rhizomeConfiguration, + authorizationService(), principalService(), searchService(), dataQueryService(), diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/V3AppUsageSurveyMigration.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/V3AppUsageSurveyMigration.kt index 13d2e1c8..01f557b5 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/V3AppUsageSurveyMigration.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/V3AppUsageSurveyMigration.kt @@ -2,7 +2,7 @@ package com.openlattice.mechanic.upgrades import com.geekbeast.postgres.PostgresArrays import com.geekbeast.rhizome.configuration.RhizomeConfiguration -import com.openlattice.authorization.Principal +import com.openlattice.authorization.* import com.openlattice.data.requests.NeighborEntityDetails import com.openlattice.data.storage.MetadataOption import com.openlattice.data.storage.postgres.PostgresEntityDataQueryService @@ -12,7 +12,9 @@ import com.openlattice.edm.EdmConstants.Companion.LAST_WRITE_FQN import com.openlattice.graph.PagedNeighborRequest import com.openlattice.hazelcast.HazelcastMap import com.openlattice.mechanic.Toolbox +import com.openlattice.organization.PERMISSION import com.openlattice.organizations.roles.SecurePrincipalsManager +import com.openlattice.postgres.PostgresColumn.PERMISSIONS import com.openlattice.search.SearchService import com.openlattice.search.requests.EntityNeighborsFilter import com.zaxxer.hikari.HikariConfig @@ -28,6 +30,7 @@ import java.util.* class V3AppUsageSurveyMigration( toolbox: Toolbox, private val rhizomeConfiguration: RhizomeConfiguration, + private val authorizationService: AuthorizationManager, private val principalService: SecurePrincipalsManager, private val searchService: SearchService, private val dataQueryService: PostgresEntityDataQueryService, @@ -238,7 +241,8 @@ class V3AppUsageSurveyMigration( val participantEntitySetIds = when (orgId) { LEGACY_ORG_ID -> getLegacyParticipantEntitySetIds(studyIds) else -> setOf(participantsEntitySetId!!) - } + }.filter { authorizationService.checkIfHasPermissions(AclKey(it), principals, EnumSet.of(Permission.READ)) }.toSet() + val participants = getStudyParticipants(studiesEntitySetId, studyEntityKeyIds, participantEntitySetIds, participatedInEntitySetId, principals) val participantsByStudyId = participants.mapValues { (_, neighbor) -> neighbor.map { it.neighborId.get() }.toSet() } From b9b50e02de8730078c736c70b0592c493b85cacd Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Mon, 21 Feb 2022 16:57:31 -0800 Subject: [PATCH 15/99] handle case where there multiple ol.title values --- .../mechanic/upgrades/V3AppUsageSurveyMigration.kt | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/V3AppUsageSurveyMigration.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/V3AppUsageSurveyMigration.kt index 01f557b5..4653c0d6 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/V3AppUsageSurveyMigration.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/V3AppUsageSurveyMigration.kt @@ -167,8 +167,13 @@ class V3AppUsageSurveyMigration( } val submissionDate = getFirstValueOrNull(entity, LAST_WRITE_FQN) - val applicationLabel = getFirstValueOrNull(entity, TITLE_FQN) + val applicationLabels = getAllValuesOrNull(entity, TITLE_FQN) val appPackageName = getFirstValueOrNull(entity, FULL_NAME_FQN) + val applicationLabel = when(applicationLabels.size) { + 0 -> appPackageName + 1 -> applicationLabels.first() + else -> applicationLabels.find { it != appPackageName } + } val timezone = getFirstValueOrNull(entity, TIMEZONE_FQN) if (submissionDate == null || appPackageName == null) { @@ -355,6 +360,13 @@ class V3AppUsageSurveyMigration( } } + private fun getAllValuesOrNull(entity: Map>, fqn: FullQualifiedName): Set { + entity[fqn]?.let { it -> + return it.mapNotNull { it.toString() }.toSet() + } + return setOf() + } + private fun getFirstValueOrNull(entity: Map>, fqn: FullQualifiedName): String? { entity[fqn]?.iterator()?.let { if (it.hasNext()) return it.next().toString() From 09838ebdb636a8439bc925dbec9ff63f48d8db75 Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Mon, 21 Feb 2022 21:13:22 -0800 Subject: [PATCH 16/99] write to chronicle --- .../upgrades/V3AppUsageSurveyMigration.kt | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/V3AppUsageSurveyMigration.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/V3AppUsageSurveyMigration.kt index 4653c0d6..40604d2d 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/V3AppUsageSurveyMigration.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/V3AppUsageSurveyMigration.kt @@ -123,11 +123,7 @@ class V3AppUsageSurveyMigration( } init { - // TODO: change chronicle to alpr - val (hikariConfiguration) = rhizomeConfiguration.datasourceConfigurations["alpr"]!! - val hc = HikariConfig(hikariConfiguration) - val hds = HikariDataSource(hc) - + val hds = getDatasource() hds.connection.createStatement().use { stmt -> stmt.execute(CREATE_APP_USAGE_SURVEY_TABLE_SQL) } @@ -138,11 +134,7 @@ class V3AppUsageSurveyMigration( logger.info("migrating app usage surveys to v3") - // TODO: change chronicle to alpr - val (hikariConfiguration) = rhizomeConfiguration.datasourceConfigurations["alpr"]!! - val hc = HikariConfig(hikariConfiguration) - val hds = HikariDataSource(hc) - + val hds = getDatasource() try { val written = hds.connection.use { connection -> connection.prepareStatement(INSERT_INTO_APP_USAGE_SQL).use { ps -> @@ -204,6 +196,12 @@ class V3AppUsageSurveyMigration( } } + private fun getDatasource(): HikariDataSource { + val (hikariConfiguration) = rhizomeConfiguration.datasourceConfigurations["chronicle"]!! + val hc = HikariConfig(hikariConfiguration) + return HikariDataSource(hc) + } + private fun getAllAppUsageSurveyData() { val v2DataCollectionOrgIds = appConfigs.keys .filter { it.appId == DATA_COLLECTION_APP_ID } From bb04632181b3d049ee66ca78e1ee46498b6ea092 Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Wed, 23 Feb 2022 15:54:07 -0800 Subject: [PATCH 17/99] mechanic job to export participant stats --- .../mechanic/pods/MechanicUpgradePod.kt | 14 + .../MigrateChronicleParticipantStats.kt | 484 ++++++++++++++++++ 2 files changed, 498 insertions(+) create mode 100644 src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt diff --git a/src/main/kotlin/com/openlattice/mechanic/pods/MechanicUpgradePod.kt b/src/main/kotlin/com/openlattice/mechanic/pods/MechanicUpgradePod.kt index 3767ea48..1a950652 100644 --- a/src/main/kotlin/com/openlattice/mechanic/pods/MechanicUpgradePod.kt +++ b/src/main/kotlin/com/openlattice/mechanic/pods/MechanicUpgradePod.kt @@ -72,6 +72,7 @@ import com.openlattice.linking.graph.PostgresLinkingQueryService import com.openlattice.mechanic.MechanicCli.Companion.UPGRADE import com.openlattice.mechanic.Toolbox import com.openlattice.mechanic.upgrades.DeleteOrgMetadataEntitySets +import com.openlattice.mechanic.upgrades.MigrateChronicleParticipantStats import com.openlattice.mechanic.upgrades.V3StudyMigrationUpgrade import com.openlattice.organizations.roles.HazelcastPrincipalService import com.openlattice.organizations.roles.SecurePrincipalsManager @@ -379,6 +380,19 @@ class MechanicUpgradePod { ) } + @Bean + fun migrateChronicleParticipantStats(): MigrateChronicleParticipantStats { + return MigrateChronicleParticipantStats( + toolbox, + searchService(), + principalService(), + entitySetService(), + dataQueryService(), + rhizomeConfiguration, + authorizationService() + ) + } + @PostConstruct fun post() { Principals.init(principalService(), hazelcastInstance) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt new file mode 100644 index 00000000..c6d755dc --- /dev/null +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt @@ -0,0 +1,484 @@ +package com.openlattice.mechanic.upgrades + +import com.geekbeast.rhizome.configuration.RhizomeConfiguration +import com.openlattice.authorization.AclKey +import com.openlattice.authorization.AuthorizationManager +import com.openlattice.authorization.Permission +import com.openlattice.authorization.Principal +import com.openlattice.data.requests.NeighborEntityDetails +import com.openlattice.data.storage.postgres.PostgresEntityDataQueryService +import com.openlattice.datastore.services.EntitySetManager +import com.openlattice.edm.EdmConstants +import com.openlattice.graph.PagedNeighborRequest +import com.openlattice.hazelcast.HazelcastMap +import com.openlattice.mechanic.Toolbox +import com.openlattice.organizations.roles.SecurePrincipalsManager +import com.openlattice.search.SearchService +import com.openlattice.search.requests.EntityNeighborsFilter +import com.zaxxer.hikari.HikariConfig +import com.zaxxer.hikari.HikariDataSource +import org.apache.olingo.commons.api.edm.FullQualifiedName +import org.slf4j.LoggerFactory +import java.time.OffsetDateTime +import java.util.* + +/** + * @author alfoncenzioka <alfonce@openlattice.com> + */ +class MigrateChronicleParticipantStats( + val toolbox: Toolbox, + private val searchService: SearchService, + private val principalService: SecurePrincipalsManager, + private val entitySetService: EntitySetManager, + private val dataQueryService: PostgresEntityDataQueryService, + private val rhizomeConfiguration: RhizomeConfiguration, + private val authorizationService: AuthorizationManager +) : Upgrade { + + // entitySetName -> string + private val entitySetIds: Map = HazelcastMap.ENTITY_SETS.getMap(toolbox.hazelcast).associate { it.value.name to it.key } + + private val entities: MutableList = mutableListOf() + + companion object { + private val logger = LoggerFactory.getLogger(MigrateChronicleParticipantStats::class.java) + + private val chronicleSuperUserIds = setOf("auth0|5ae9026c04eb0b243f1d2bb6", "google-oauth2|113860246540203337319") + + private val LEGACY_ORG_ID = UUID.fromString("7349c446-2acc-4d14-b2a9-a13be39cff93") + private val DATA_COLLECTION_APP_ID = UUID.fromString("c4e6d8fd-daf9-41e7-8c59-2a12c7ee0857") + private val SURVEY_APP_ID = UUID.fromString("bb44218b-515a-4314-b955-df2c991b2575") + + private const val CHRONICLE_APP_ES_PREFIX = "chronicle_" + private const val SURVEYS_APP_ES_PREFIX = "chronicle_surveys_" + + // collection template names + private const val STUDIES_TEMPLATE = "studies" + private const val PARTICIPATED_IN_TEMPLATE = "participatedin" + private const val RESPONDS_WITH_TEMPLATE = "respondswith" + private const val SUBMISSION_TEMPLATE = "submission" + private const val PARTICIPANTS_TEMPLATE = "participants" + private const val HAS_TEMPLATE = "has" + private const val METADATA_TEMPLATE = "metadata" + + // legacy entity set names + private const val LEGACY_STUDIES_ES = "chronicle_study" + private const val LEGACY_PARTICIPATED_IN_ES = "chronicle_participated_in" + private const val LEGACY_RECORDED_BY_ES = "chronicle_recorded_by" + private const val LEGACY_HAS_ES = "chronicle_has" + private const val LEGACY_METADATA_ES = "chronicle_metadata" + + // entity sets lookup name + private const val STUDIES_ES = "studies" + private const val PARTICIPATED_IN_ES = "participatedIn" + private const val PARTICIPANTS_ES = "participants" + private const val SUBMISSION_ES = "submission" + private const val RESPONDS_WITH_ES = "respondsWith" + private const val HAS_ES = "has" + private const val METADATA_ES = "metadata" + + private val OL_ID_FQN = EdmConstants.ID_FQN + private val STRING_ID_FQN = FullQualifiedName("general.stringid") + private val PERSON_FQN = FullQualifiedName("nc.SubjectIdentification") + private val FULL_NAME_FQN = FullQualifiedName("general.fullname") + private val DATE_TIME_START_FQN = FullQualifiedName("ol.datetimestart") + private val DATE_TIME_END_FQN = FullQualifiedName("ol.datetimeend") + private val DATETIME_FQN = FullQualifiedName("ol.datetime") + private val RECORDED_DATE_FQN = FullQualifiedName("ol.recordeddate") + + // column names + private const val STUDY_ID = "study_id" + private const val PARTICIPANT_ID = "participant_id" + private const val ANDROID_FIRST_DATE = "android_first_date" + private const val ANDROID_LAST_DATE = "android_last_date" + private const val ANDROID_DATES_COUNT = "android_dates_count" + private const val IOS_FIRST_DATE = "ios_first_date" + private const val IOS_LAST_DATE = "ios_last_date" + private const val IOS_DATES_COUNT = "ios_dates_count" + private const val TUD_FIRST_DATE = "tud_first_date" + private const val TUD_LAST_DATE = "tud_last_date" + private const val TUD_DATES_COUNT = "tud_dates_count" + + private val CREATE_STATS_TABLE_SQL = """ + CREATE TABLE IF NOT EXISTS participant_stats( + $STUDY_ID uuid NOT NULL, + $PARTICIPANT_ID text NOT NULL, + $ANDROID_FIRST_DATE timestamp with time zone, + $ANDROID_LAST_DATE timestamp with time zone, + $ANDROID_DATES_COUNT integer, + $IOS_FIRST_DATE timestamp with time zone, + $IOS_LAST_DATE timestamp with time zone, + $IOS_DATES_COUNT integer, + $TUD_FIRST_DATE timestamp with time zone, + $TUD_LAST_DATE timestamp with time zone, + $TUD_DATES_COUNT integer, + PRIMARY KEY($STUDY_ID, $PARTICIPANT_ID) + ) + """.trimIndent() + + private val COLS = setOf( + STUDY_ID, + PARTICIPANT_ID, + ANDROID_FIRST_DATE, + ANDROID_LAST_DATE, + ANDROID_DATES_COUNT, + TUD_FIRST_DATE, + TUD_LAST_DATE, + TUD_DATES_COUNT + ) + + private val PARTICIPANT_STATS_COLS = COLS.joinToString { it } + private val PARTICIPANT_STATS_PARAMS = COLS.joinToString { "?" } + + /**PreparedStatement bind order + * 1) studyId, + * 2) participantId + * 3) androidFirstDate + * 4) androidLastDate + * 5) androidDatesCount, + * 6) tudFirstDate, + * 7) tudLastDate + * 8) tudDatesCount + */ + private val INSERT_PARTICIPANT_STATS_SQL = """ + INSERT INTO participant_stats ($PARTICIPANT_STATS_COLS) values ($PARTICIPANT_STATS_PARAMS) + """.trimIndent() + } + + init { + // create table + val hds = getHikariDataSource() + hds.connection.createStatement().use { stmt -> stmt.execute(CREATE_STATS_TABLE_SQL) } + } + + + override fun upgrade(): Boolean { + getEntitiesToInsert() + val totalWritten = writeEntitiesToTable() + logger.info("Exported $totalWritten entities to participant stats table. Expected to export ${entities.size} entities") + return true + } + + + override fun getSupportedVersion(): Long { + return Version.V2021_07_23.value + } + + private fun getEntitiesToInsert() { + + val orgIdsByAppId = getOrgIdsByAppId().toMutableMap() + orgIdsByAppId.getValue(DATA_COLLECTION_APP_ID).add(LEGACY_ORG_ID) + + val superUserPrincipals = getChronicleSuperUserPrincipals() + + (orgIdsByAppId.values.flatten().toSet()).forEach { orgId -> + logger.info("---------------------------------------------") + logger.info("Retrieving entities in organization $orgId") + logger.info("----------------------------------------------") + + val entitySets = getOrgEntitySetNames(orgId) + + // step 1: get studies in org + val studies: Map = getOrgStudies(entitySetId = entitySets.getValue(STUDIES_ES)) + if (studies.isEmpty()) { + logger.info("organization ${orgId} has no studies. Skipping") + return@forEach + } + logger.info("Retrieved ${studies.size} studies belonging to org $orgId") + + // step 2: get all participants in org + val legacyParticipantEntitySets = getLegacyParticipantEntitySetIds(studies.values.map { it.generalStringId }.toSet()) + val participantEntitySets = when (orgId) { + LEGACY_ORG_ID -> legacyParticipantEntitySets + else -> setOf(entitySets.getValue(PARTICIPANTS_ES)) + }.filter { authorizationService.checkIfHasPermissions(AclKey(it), superUserPrincipals, EnumSet.of(Permission.READ)) }.toSet() + + val participants = getOrgParticipants( + participantEntitySetIds = participantEntitySets, + studiesEntitySetId = entitySets.getValue(STUDIES_ES), + entityKeyIds = studies.keys, + principals = superUserPrincipals, + edgeEntitySetId = entitySets.getValue(PARTICIPATED_IN_ES) + ) + logger.info("Retrieved ${participants.values.flatten().size} belonging to org $orgId") + logger.info("Participant count by study: ${participants.map { studies.getValue(it.key).title to it.value.size }.toMap()}") + + // step 3: neighbor search on participant entity set + val participantStats = getParticipantStats( + participantEntitySets = participantEntitySets, + entitySetIds = entitySets, + entityKeyIds = participants.values.flatten().map { it.id }.toSet(), + orgIdsByAppId = orgIdsByAppId, + orgId = orgId, + principals = superUserPrincipals, + participantById = participants.values.flatten().associateBy { it.id }, + studyIdByParticipantId = participants.values.flatten().associate { it.id to it.studyId } + ) + + logger.info("Participant stats entities by study: ${participantStats.map { studies.getValue(it.key).title to it.value.size }.toMap()}") + entities.addAll(participantStats.values.flatten()) + } + + logger.info("Total entities to write: ${entities.size}") + } + + private fun writeEntitiesToTable(): Int { + return getHikariDataSource().connection.use { connection -> + try { + val wc = connection.prepareStatement(INSERT_PARTICIPANT_STATS_SQL).use { ps -> + entities.forEach { + var index = 0 + ps.setObject(++index, it.studyId) + ps.setString(++index, it.participantId) + ps.setObject(++index, it.androidFirstDate) + ps.setObject(++index, it.androidLastDate) + ps.setInt(++index, it.androidDatesCount) + ps.setObject(++index, it.tudFirstDate) + ps.setObject(++index, it.tudLastDate) + ps.setObject(++index, it.tudDatesCount) + ps.addBatch() + } + ps.executeBatch().sum() + } + return@use wc + } catch (ex: Exception) { + throw ex + } + } + } + + private fun getOrgStudies(entitySetId: UUID): Map { + return dataQueryService.getEntitiesWithPropertyTypeFqns( + mapOf(entitySetId to Optional.empty()), + entitySetService.getPropertyTypesOfEntitySets(setOf(entitySetId)), + mapOf(), + setOf(), + Optional.empty(), + false + ) + .filter { getFirstUUIDOrNull(it.value, STRING_ID_FQN) != null } + .mapValues { getStudyEntity(it.key, it.value) } + } + + // Returns a mapping from studyId to list of participants + private fun getOrgParticipants( + participantEntitySetIds: Set, + edgeEntitySetId: UUID, + studiesEntitySetId: UUID, + entityKeyIds: Set, + principals: Set + ) + : Map> { + val filter = EntityNeighborsFilter(entityKeyIds, Optional.of(participantEntitySetIds), Optional.empty(), Optional.of(setOf(edgeEntitySetId))) + + return searchService + .executeEntityNeighborSearch(setOf(studiesEntitySetId), PagedNeighborRequest(filter), principals) + .neighbors + .mapValues { it.value.map { neighbor -> getParticipantFromNeighborEntity(it.key, neighbor) } } + + } + + // mapping from studyId to a list of participant stats objects + private fun getParticipantStats( + participantEntitySets: Set, + entitySetIds: Map, + entityKeyIds: Set, + orgIdsByAppId: Map>, + orgId: UUID, + principals: Set, + studyIdByParticipantId: Map, + participantById: Map + ): Map> { + + val srcEntitySetIds: MutableSet = participantEntitySets.toMutableSet() + val edgeEntitySetIds: MutableSet = mutableSetOf(entitySetIds.getValue(HAS_ES)) + val dstEntitySetIds: MutableSet = mutableSetOf(entitySetIds.getValue(METADATA_ES)) + + if (isAppIdInOrg(orgId, SURVEY_APP_ID, orgIdsByAppId)) { + edgeEntitySetIds.add(entitySetIds.getValue(RESPONDS_WITH_ES)) + dstEntitySetIds.add(entitySetIds.getValue(SUBMISSION_ES)) + } + + val filter = EntityNeighborsFilter( + entityKeyIds, + Optional.of(srcEntitySetIds), + Optional.of(dstEntitySetIds), + Optional.of(edgeEntitySetIds) + ) + + return searchService.executeEntityNeighborSearch(participantEntitySets, PagedNeighborRequest(filter), principals) + .neighbors + .mapValues { (id, neighbors) -> + val neighborsByAssociationES = neighbors.groupBy { it.associationEntitySet.id } + val androidStats = getParticipantAndroidStats(neighborsByAssociationES[entitySetIds.getValue(HAS_ES)]) + val tudStats = getParticipantTudStats(neighborsByAssociationES[entitySetIds.getValue(RESPONDS_WITH_ES)]) + + ParticipantStats( + studyId = studyIdByParticipantId.getValue(id), + participantId = participantById.getValue(id).participantId, + androidFirstDate = androidStats.first, + androidLastDate = androidStats.second, + androidDatesCount = androidStats.third, + tudFirstDate = tudStats.first, + tudLastDate = tudStats.second, + tudDatesCount = tudStats.third + ) + }.values.groupBy { it.studyId } + } + + // start, end date, count + // in theory each participant should only have a single NeighborEntityDetails in the metadata entity set, + // but some might have multiple entities + private fun getParticipantAndroidStats(neighbors: List?): Triple { + + if (neighbors == null || neighbors.isEmpty()) { + return Triple(null, null, 0) + } + + val dateTimeStartValues = getOffsetDateTimesFromNeighborEntities(neighbors, DATE_TIME_START_FQN) + val dateTimeEndValues = getOffsetDateTimesFromNeighborEntities(neighbors, DATE_TIME_END_FQN) + val datesRecorded = getOffsetDateTimesFromNeighborEntities(neighbors, RECORDED_DATE_FQN) + + return Triple( + first = dateTimeStartValues.stream().min(OffsetDateTime::compareTo).get(), + second = dateTimeEndValues.stream().max(OffsetDateTime::compareTo).get(), + third = datesRecorded.map { it.toLocalDate() }.toSet().size // unique dates + ) + } + + // start date, end date, count + private fun getParticipantTudStats(neighbors: List?): Triple { + if (neighbors == null) return Triple(null, null, 0) + + val dateTimeValues = getOffsetDateTimesFromNeighborEntities(neighbors, DATETIME_FQN) + + return Triple( + first = dateTimeValues.stream().min(OffsetDateTime::compareTo).get(), + second = dateTimeValues.stream().max(OffsetDateTime::compareTo).get(), + third = dateTimeValues.map { it.toLocalDate() }.toSet().size // unique dates + ) + } + + // returns a mapping from appId to setOf organizations containing app + private fun getOrgIdsByAppId(): Map> { + return HazelcastMap.APP_CONFIGS.getMap(toolbox.hazelcast).keys + .filter { it.appId == DATA_COLLECTION_APP_ID || it.appId == SURVEY_APP_ID } + .groupBy { it.appId } + .mapValues { it.value.map { config -> config.organizationId }.toMutableSet() } + } + + + private fun getOffsetDateTimesFromNeighborEntities(entities: List, fqn: FullQualifiedName): Set { + return entities + .map { getAllValuesOrNull(it.neighborDetails.get(), fqn) } + .flatten().map { OffsetDateTime.parse(it) }.toSet() + } + + private fun isAppIdInOrg(orgId: UUID, appId: UUID, orgIdsByAppId: Map>): Boolean { + return orgIdsByAppId.getValue(appId).contains(orgId) + } + + private fun getChronicleSuperUserPrincipals(): Set { + return chronicleSuperUserIds + .map { principalService.getSecurablePrincipal(it) } + .map { principalService.getAllPrincipals(it).map { principal -> principal.principal } } + .flatten().toSet() + } + + private fun getLegacyParticipantEntitySetIds(studyIds: Set): Set { + val entitySetNames = studyIds.map { "chronicle_participants_$it" } + return entitySetNames.mapNotNull { entitySetIds[it] }.toSet() + } + + private fun getOrgEntitySetNames(orgId: UUID): Map { + val entitySetNameByTemplateName = when (orgId) { + LEGACY_ORG_ID -> mapOf( + STUDIES_ES to LEGACY_STUDIES_ES, + PARTICIPATED_IN_ES to LEGACY_PARTICIPATED_IN_ES, + HAS_ES to LEGACY_HAS_ES, + METADATA_ES to LEGACY_METADATA_ES + ) + else -> { + val orgIdToStr = orgId.toString().replace("-", "") + mapOf( + STUDIES_ES to "$CHRONICLE_APP_ES_PREFIX${orgIdToStr}_$STUDIES_TEMPLATE", + HAS_ES to "$CHRONICLE_APP_ES_PREFIX${orgIdToStr}_$HAS_TEMPLATE", + METADATA_ES to "$CHRONICLE_APP_ES_PREFIX${orgIdToStr}_$METADATA_TEMPLATE", + PARTICIPATED_IN_ES to "$CHRONICLE_APP_ES_PREFIX${orgIdToStr}_$PARTICIPATED_IN_TEMPLATE", + PARTICIPANTS_ES to "$CHRONICLE_APP_ES_PREFIX${orgIdToStr}_${PARTICIPANTS_TEMPLATE}", + SUBMISSION_ES to "$SURVEYS_APP_ES_PREFIX${orgIdToStr}_${SUBMISSION_TEMPLATE}", + RESPONDS_WITH_ES to "$SURVEYS_APP_ES_PREFIX${orgIdToStr}_${RESPONDS_WITH_TEMPLATE}", + ) + } + } + + return entitySetNameByTemplateName.mapValues { entitySetIds.getValue(it.value) } + } + + private fun getParticipantFromNeighborEntity(studyId: UUID, entity: NeighborEntityDetails): Participant { + val id = getFirstUUIDOrNull(entity.neighborDetails.get(), OL_ID_FQN) + val participantId = getFirstValueOrNull(entity.neighborDetails.get(), PERSON_FQN) + + return Participant(studyId, id!!, participantId!!) // hope this force unwrapping doesn't throw NPE + + } + + + + private fun getFirstUUIDOrNull(entity: Map>, fqn: FullQualifiedName): UUID? { + return when (val string = getFirstValueOrNull(entity, fqn)) { + null -> null + else -> UUID.fromString(string) + } + } + + private fun getAllValuesOrNull(entity: Map>, fqn: FullQualifiedName): Set { + entity[fqn]?.let { it -> + return it.mapNotNull { it.toString() }.toSet() + } + return setOf() + } + + private fun getFirstValueOrNull(entity: Map>, fqn: FullQualifiedName): String? { + entity[fqn]?.iterator()?.let { + if (it.hasNext()) return it.next().toString() + } + return null + } + + private fun getStudyEntity(studyId: UUID, entity: Map>): Study { + val title = getFirstValueOrNull(entity, FULL_NAME_FQN) + val generalStringId = getFirstUUIDOrNull(entity, STRING_ID_FQN) + return Study(studyId, generalStringId!!, title) + } + + + private fun getHikariDataSource(): HikariDataSource { + val (hikariConfiguration) = rhizomeConfiguration.datasourceConfigurations["alpr"]!! + val hc = HikariConfig(hikariConfiguration) + return HikariDataSource(hc) + } +} + +private data class Participant( + val studyId: UUID, + val id: UUID, + val participantId: String, +) + +private data class ParticipantStats( + val studyId: UUID, + val participantId: String, + val androidFirstDate: Any?, + val androidLastDate: Any?, + val androidDatesCount: Int = 0, + val tudFirstDate: Any?, + val tudLastDate: Any?, + val tudDatesCount: Int = 0 +) + +private data class Study( + val studyId: UUID, + val generalStringId: UUID, + val title: String? +) \ No newline at end of file From dcbb677226e8d9a8b31f1c27954e99be223d5c50 Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Wed, 23 Feb 2022 16:26:23 -0800 Subject: [PATCH 18/99] search serviceuse principals with admin role --- .../MigrateChronicleParticipantStats.kt | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt index c6d755dc..4d60007f 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt @@ -37,6 +37,7 @@ class MigrateChronicleParticipantStats( // entitySetName -> string private val entitySetIds: Map = HazelcastMap.ENTITY_SETS.getMap(toolbox.hazelcast).associate { it.value.name to it.key } + private val organizations = HazelcastMap.ORGANIZATIONS.getMap(toolbox.hazelcast) private val entities: MutableList = mutableListOf() @@ -169,13 +170,19 @@ class MigrateChronicleParticipantStats( val orgIdsByAppId = getOrgIdsByAppId().toMutableMap() orgIdsByAppId.getValue(DATA_COLLECTION_APP_ID).add(LEGACY_ORG_ID) - val superUserPrincipals = getChronicleSuperUserPrincipals() - (orgIdsByAppId.values.flatten().toSet()).forEach { orgId -> logger.info("---------------------------------------------") logger.info("Retrieving entities in organization $orgId") logger.info("----------------------------------------------") + // get principals + val adminRoleAclKey = organizations[orgId]?.adminRoleAclKey + if (adminRoleAclKey == null) { + logger.warn("skipping {} since it doesn't have admin role", orgId) + return@forEach + } + val principals = principalService.getAllUsersWithPrincipal(adminRoleAclKey).map { it.principal }.toSet() + getChronicleSuperUserPrincipals() + val entitySets = getOrgEntitySetNames(orgId) // step 1: get studies in org @@ -191,13 +198,13 @@ class MigrateChronicleParticipantStats( val participantEntitySets = when (orgId) { LEGACY_ORG_ID -> legacyParticipantEntitySets else -> setOf(entitySets.getValue(PARTICIPANTS_ES)) - }.filter { authorizationService.checkIfHasPermissions(AclKey(it), superUserPrincipals, EnumSet.of(Permission.READ)) }.toSet() + }.filter { authorizationService.checkIfHasPermissions(AclKey(it), principals, EnumSet.of(Permission.READ)) }.toSet() val participants = getOrgParticipants( participantEntitySetIds = participantEntitySets, studiesEntitySetId = entitySets.getValue(STUDIES_ES), entityKeyIds = studies.keys, - principals = superUserPrincipals, + principals = principals, edgeEntitySetId = entitySets.getValue(PARTICIPATED_IN_ES) ) logger.info("Retrieved ${participants.values.flatten().size} belonging to org $orgId") @@ -210,7 +217,7 @@ class MigrateChronicleParticipantStats( entityKeyIds = participants.values.flatten().map { it.id }.toSet(), orgIdsByAppId = orgIdsByAppId, orgId = orgId, - principals = superUserPrincipals, + principals = principals, participantById = participants.values.flatten().associateBy { it.id }, studyIdByParticipantId = participants.values.flatten().associate { it.id to it.studyId } ) From 2b64e5c5c90f51521da1661abd556bfa32b82572 Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Wed, 23 Feb 2022 18:26:58 -0800 Subject: [PATCH 19/99] guard against nulls --- .../mechanic/upgrades/MigrateChronicleParticipantStats.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt index 4d60007f..c61c8ad6 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt @@ -419,7 +419,7 @@ class MigrateChronicleParticipantStats( } } - return entitySetNameByTemplateName.mapValues { entitySetIds.getValue(it.value) } + return entitySetNameByTemplateName.filter { entitySetIds.keys.contains(it.value) }.mapValues { entitySetIds.getValue(it.value) } } private fun getParticipantFromNeighborEntity(studyId: UUID, entity: NeighborEntityDetails): Participant { From 6f83c87accc58fb7172966ec5acf03e70dc81aab Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Wed, 23 Feb 2022 19:10:44 -0800 Subject: [PATCH 20/99] fix NPE --- .../mechanic/upgrades/MigrateChronicleParticipantStats.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt index c61c8ad6..a9db857d 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt @@ -318,7 +318,7 @@ class MigrateChronicleParticipantStats( .mapValues { (id, neighbors) -> val neighborsByAssociationES = neighbors.groupBy { it.associationEntitySet.id } val androidStats = getParticipantAndroidStats(neighborsByAssociationES[entitySetIds.getValue(HAS_ES)]) - val tudStats = getParticipantTudStats(neighborsByAssociationES[entitySetIds.getValue(RESPONDS_WITH_ES)]) + val tudStats = getParticipantTudStats(neighborsByAssociationES[entitySetIds[RESPONDS_WITH_ES]]) // not every org has respondsWith entity set ParticipantStats( studyId = studyIdByParticipantId.getValue(id), From 2613d7c6b47a3f2273868ea6e1ffea10c7cb899a Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Thu, 24 Feb 2022 06:33:44 -0800 Subject: [PATCH 21/99] check for duplicates --- .../MigrateChronicleParticipantStats.kt | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt index a9db857d..6d660386 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt @@ -165,6 +165,13 @@ class MigrateChronicleParticipantStats( return Version.V2021_07_23.value } + + private fun getHikariDataSource(): HikariDataSource { + val (hikariConfiguration) = rhizomeConfiguration.datasourceConfigurations["alpr"]!! + val hc = HikariConfig(hikariConfiguration) + return HikariDataSource(hc) + } + private fun getEntitiesToInsert() { val orgIdsByAppId = getOrgIdsByAppId().toMutableMap() @@ -210,6 +217,16 @@ class MigrateChronicleParticipantStats( logger.info("Retrieved ${participants.values.flatten().size} belonging to org $orgId") logger.info("Participant count by study: ${participants.map { studies.getValue(it.key).title to it.value.size }.toMap()}") + + // check for duplicate participant ids. for each study, participantIds should be unique + participants.forEach { (studyId, participants) -> + val distinctParticipants = participants.distinct() + val difference = participants - distinctParticipants.toSet() + if (difference.isNotEmpty()) { + logger.info("Found duplicate participant ids in ${studies.getValue(studyId)} in org $orgId: $difference") + } + } + // step 3: neighbor search on participant entity set val participantStats = getParticipantStats( participantEntitySets = participantEntitySets, @@ -224,6 +241,7 @@ class MigrateChronicleParticipantStats( logger.info("Participant stats entities by study: ${participantStats.map { studies.getValue(it.key).title to it.value.size }.toMap()}") entities.addAll(participantStats.values.flatten()) + } logger.info("Total entities to write: ${entities.size}") @@ -458,13 +476,6 @@ class MigrateChronicleParticipantStats( val generalStringId = getFirstUUIDOrNull(entity, STRING_ID_FQN) return Study(studyId, generalStringId!!, title) } - - - private fun getHikariDataSource(): HikariDataSource { - val (hikariConfiguration) = rhizomeConfiguration.datasourceConfigurations["alpr"]!! - val hc = HikariConfig(hikariConfiguration) - return HikariDataSource(hc) - } } private data class Participant( From e7fb2c5dad97854783d7efbb7ae9f17c0908a4f9 Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Thu, 24 Feb 2022 07:30:33 -0800 Subject: [PATCH 22/99] assert --- .../upgrades/MigrateChronicleParticipantStats.kt | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt index 6d660386..dbe8a1b4 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt @@ -19,6 +19,7 @@ import com.zaxxer.hikari.HikariConfig import com.zaxxer.hikari.HikariDataSource import org.apache.olingo.commons.api.edm.FullQualifiedName import org.slf4j.LoggerFactory +import org.springframework.util.Assert import java.time.OffsetDateTime import java.util.* @@ -195,10 +196,10 @@ class MigrateChronicleParticipantStats( // step 1: get studies in org val studies: Map = getOrgStudies(entitySetId = entitySets.getValue(STUDIES_ES)) if (studies.isEmpty()) { - logger.info("organization ${orgId} has no studies. Skipping") + logger.info("organization $orgId has no studies. Skipping") return@forEach } - logger.info("Retrieved ${studies.size} studies belonging to org $orgId") + logger.info("Retrieved ${studies.size} studies in org $orgId") // step 2: get all participants in org val legacyParticipantEntitySets = getLegacyParticipantEntitySetIds(studies.values.map { it.generalStringId }.toSet()) @@ -214,7 +215,7 @@ class MigrateChronicleParticipantStats( principals = principals, edgeEntitySetId = entitySets.getValue(PARTICIPATED_IN_ES) ) - logger.info("Retrieved ${participants.values.flatten().size} belonging to org $orgId") + logger.info("Retrieved ${participants.values.flatten().size} participants in org $orgId") logger.info("Participant count by study: ${participants.map { studies.getValue(it.key).title to it.value.size }.toMap()}") @@ -238,6 +239,8 @@ class MigrateChronicleParticipantStats( participantById = participants.values.flatten().associateBy { it.id }, studyIdByParticipantId = participants.values.flatten().associate { it.id to it.studyId } ) + + assert(participants.values.flatten().associateBy { it.id }.keys.size == participants.values.flatten().map { it.id }.toSet().size) logger.info("Participant stats entities by study: ${participantStats.map { studies.getValue(it.key).title to it.value.size }.toMap()}") entities.addAll(participantStats.values.flatten()) From 506200b5fa390633c254e173bcd0ca09f0f773c0 Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Thu, 24 Feb 2022 07:35:07 -0800 Subject: [PATCH 23/99] remove unneeded --- .../mechanic/upgrades/MigrateChronicleParticipantStats.kt | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt index dbe8a1b4..fd1df7c6 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt @@ -232,15 +232,12 @@ class MigrateChronicleParticipantStats( val participantStats = getParticipantStats( participantEntitySets = participantEntitySets, entitySetIds = entitySets, - entityKeyIds = participants.values.flatten().map { it.id }.toSet(), orgIdsByAppId = orgIdsByAppId, orgId = orgId, principals = principals, participantById = participants.values.flatten().associateBy { it.id }, studyIdByParticipantId = participants.values.flatten().associate { it.id to it.studyId } ) - - assert(participants.values.flatten().associateBy { it.id }.keys.size == participants.values.flatten().map { it.id }.toSet().size) logger.info("Participant stats entities by study: ${participantStats.map { studies.getValue(it.key).title to it.value.size }.toMap()}") entities.addAll(participantStats.values.flatten()) @@ -310,7 +307,6 @@ class MigrateChronicleParticipantStats( private fun getParticipantStats( participantEntitySets: Set, entitySetIds: Map, - entityKeyIds: Set, orgIdsByAppId: Map>, orgId: UUID, principals: Set, @@ -328,7 +324,7 @@ class MigrateChronicleParticipantStats( } val filter = EntityNeighborsFilter( - entityKeyIds, + participantById.keys, Optional.of(srcEntitySetIds), Optional.of(dstEntitySetIds), Optional.of(edgeEntitySetIds) From 689ff03238502c293cfe42d6ef43fe533252d2d4 Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Thu, 24 Feb 2022 07:47:40 -0800 Subject: [PATCH 24/99] format --- .../mechanic/upgrades/MigrateChronicleParticipantStats.kt | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt index fd1df7c6..26b3c9a5 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt @@ -19,7 +19,6 @@ import com.zaxxer.hikari.HikariConfig import com.zaxxer.hikari.HikariDataSource import org.apache.olingo.commons.api.edm.FullQualifiedName import org.slf4j.LoggerFactory -import org.springframework.util.Assert import java.time.OffsetDateTime import java.util.* @@ -447,8 +446,6 @@ class MigrateChronicleParticipantStats( } - - private fun getFirstUUIDOrNull(entity: Map>, fqn: FullQualifiedName): UUID? { return when (val string = getFirstValueOrNull(entity, fqn)) { null -> null From 2ace5e003d4cb4c05ce80f0576f4ec83b76180fa Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Thu, 24 Feb 2022 07:53:56 -0800 Subject: [PATCH 25/99] assert --- .../mechanic/upgrades/MigrateChronicleParticipantStats.kt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt index 26b3c9a5..35718fdf 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt @@ -237,6 +237,9 @@ class MigrateChronicleParticipantStats( participantById = participants.values.flatten().associateBy { it.id }, studyIdByParticipantId = participants.values.flatten().associate { it.id to it.studyId } ) + val participantsIds = participants.values.flatten().map { it.participantId } + val vals = participants.values.flatten() + assert(vals.size == participantsIds.size ) logger.info("Participant stats entities by study: ${participantStats.map { studies.getValue(it.key).title to it.value.size }.toMap()}") entities.addAll(participantStats.values.flatten()) From 13fa484b38c31625ed4a10069e28108fedbdad0c Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Thu, 24 Feb 2022 08:49:02 -0800 Subject: [PATCH 26/99] fix stuff --- .../MigrateChronicleParticipantStats.kt | 84 +++++++++++-------- 1 file changed, 48 insertions(+), 36 deletions(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt index 35718fdf..b708e8c0 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt @@ -88,7 +88,9 @@ class MigrateChronicleParticipantStats( private val RECORDED_DATE_FQN = FullQualifiedName("ol.recordeddate") // column names - private const val STUDY_ID = "study_id" + private const val ORGANIZATION_IO = "organization_id" + private const val V2_STUDY_ID = "v2_study_id" + private const val V2_STUDY_EKID = "v2_study_ekid" private const val PARTICIPANT_ID = "participant_id" private const val ANDROID_FIRST_DATE = "android_first_date" private const val ANDROID_LAST_DATE = "android_last_date" @@ -102,7 +104,9 @@ class MigrateChronicleParticipantStats( private val CREATE_STATS_TABLE_SQL = """ CREATE TABLE IF NOT EXISTS participant_stats( - $STUDY_ID uuid NOT NULL, + $ORGANIZATION_IO uuid NOT NULL, + $V2_STUDY_EKID uuid NOT NULL, + $V2_STUDY_ID uuid NOT NULL, $PARTICIPANT_ID text NOT NULL, $ANDROID_FIRST_DATE timestamp with time zone, $ANDROID_LAST_DATE timestamp with time zone, @@ -113,12 +117,14 @@ class MigrateChronicleParticipantStats( $TUD_FIRST_DATE timestamp with time zone, $TUD_LAST_DATE timestamp with time zone, $TUD_DATES_COUNT integer, - PRIMARY KEY($STUDY_ID, $PARTICIPANT_ID) + PRIMARY KEY($V2_STUDY_EKID, $PARTICIPANT_ID) ) """.trimIndent() private val COLS = setOf( - STUDY_ID, + ORGANIZATION_IO, + V2_STUDY_EKID, + V2_STUDY_ID, PARTICIPANT_ID, ANDROID_FIRST_DATE, ANDROID_LAST_DATE, @@ -132,14 +138,16 @@ class MigrateChronicleParticipantStats( private val PARTICIPANT_STATS_PARAMS = COLS.joinToString { "?" } /**PreparedStatement bind order - * 1) studyId, - * 2) participantId - * 3) androidFirstDate - * 4) androidLastDate - * 5) androidDatesCount, - * 6) tudFirstDate, - * 7) tudLastDate - * 8) tudDatesCount + * 1) organizationId + * 2) studyEntityKeyId, + * 3) studyId, + * 4) participantId + * 5) androidFirstDate + * 6) androidLastDate + * 7) androidDatesCount, + * 8) tudFirstDate, + * 9) tudLastDate + * 10) tudDatesCount */ private val INSERT_PARTICIPANT_STATS_SQL = """ INSERT INTO participant_stats ($PARTICIPANT_STATS_COLS) values ($PARTICIPANT_STATS_PARAMS) @@ -192,20 +200,20 @@ class MigrateChronicleParticipantStats( val entitySets = getOrgEntitySetNames(orgId) - // step 1: get studies in org + // step 1: get studies in org: {studyEntityKeyId -> study} val studies: Map = getOrgStudies(entitySetId = entitySets.getValue(STUDIES_ES)) if (studies.isEmpty()) { logger.info("organization $orgId has no studies. Skipping") return@forEach } - logger.info("Retrieved ${studies.size} studies in org $orgId") + logger.info("Retrieved ${studies.size} studies") // step 2: get all participants in org - val legacyParticipantEntitySets = getLegacyParticipantEntitySetIds(studies.values.map { it.generalStringId }.toSet()) val participantEntitySets = when (orgId) { - LEGACY_ORG_ID -> legacyParticipantEntitySets + LEGACY_ORG_ID -> getLegacyParticipantEntitySetIds(studies.values.map { it.studyId }.toSet()) else -> setOf(entitySets.getValue(PARTICIPANTS_ES)) }.filter { authorizationService.checkIfHasPermissions(AclKey(it), principals, EnumSet.of(Permission.READ)) }.toSet() + logger.info("participant entity sets: $participantEntitySets") val participants = getOrgParticipants( participantEntitySetIds = participantEntitySets, @@ -214,16 +222,16 @@ class MigrateChronicleParticipantStats( principals = principals, edgeEntitySetId = entitySets.getValue(PARTICIPATED_IN_ES) ) - logger.info("Retrieved ${participants.values.flatten().size} participants in org $orgId") + logger.info("Retrieved ${participants.values.flatten().size} participants") logger.info("Participant count by study: ${participants.map { studies.getValue(it.key).title to it.value.size }.toMap()}") // check for duplicate participant ids. for each study, participantIds should be unique - participants.forEach { (studyId, participants) -> + participants.forEach { (studyEntityKeyId, participants) -> val distinctParticipants = participants.distinct() val difference = participants - distinctParticipants.toSet() if (difference.isNotEmpty()) { - logger.info("Found duplicate participant ids in ${studies.getValue(studyId)} in org $orgId: $difference") + logger.info("Found duplicate participant ids in ${studies.getValue(studyEntityKeyId)}") } } @@ -235,11 +243,8 @@ class MigrateChronicleParticipantStats( orgId = orgId, principals = principals, participantById = participants.values.flatten().associateBy { it.id }, - studyIdByParticipantId = participants.values.flatten().associate { it.id to it.studyId } + studies = studies ) - val participantsIds = participants.values.flatten().map { it.participantId } - val vals = participants.values.flatten() - assert(vals.size == participantsIds.size ) logger.info("Participant stats entities by study: ${participantStats.map { studies.getValue(it.key).title to it.value.size }.toMap()}") entities.addAll(participantStats.values.flatten()) @@ -255,6 +260,8 @@ class MigrateChronicleParticipantStats( val wc = connection.prepareStatement(INSERT_PARTICIPANT_STATS_SQL).use { ps -> entities.forEach { var index = 0 + ps.setObject(++index, it.organizationId) + ps.setObject(++index, it.studyEntityKeyId) ps.setObject(++index, it.studyId) ps.setString(++index, it.participantId) ps.setObject(++index, it.androidFirstDate) @@ -287,7 +294,7 @@ class MigrateChronicleParticipantStats( .mapValues { getStudyEntity(it.key, it.value) } } - // Returns a mapping from studyId to list of participants + // Returns a mapping from studyEntityKeyId to list of participants private fun getOrgParticipants( participantEntitySetIds: Set, edgeEntitySetId: UUID, @@ -305,15 +312,15 @@ class MigrateChronicleParticipantStats( } - // mapping from studyId to a list of participant stats objects + // mapping from studyEntityKeyId to a list of participant stats objects private fun getParticipantStats( participantEntitySets: Set, entitySetIds: Map, orgIdsByAppId: Map>, orgId: UUID, principals: Set, - studyIdByParticipantId: Map, - participantById: Map + participantById: Map, + studies: Map, ): Map> { val srcEntitySetIds: MutableSet = participantEntitySets.toMutableSet() @@ -339,8 +346,11 @@ class MigrateChronicleParticipantStats( val androidStats = getParticipantAndroidStats(neighborsByAssociationES[entitySetIds.getValue(HAS_ES)]) val tudStats = getParticipantTudStats(neighborsByAssociationES[entitySetIds[RESPONDS_WITH_ES]]) // not every org has respondsWith entity set + val studyEntityKeyId = participantById.getValue(id).studyEntityKeyId ParticipantStats( - studyId = studyIdByParticipantId.getValue(id), + organizationId = orgId, + studyEntityKeyId = studyEntityKeyId, + studyId = studies.getValue(studyEntityKeyId).studyId, participantId = participantById.getValue(id).participantId, androidFirstDate = androidStats.first, androidLastDate = androidStats.second, @@ -349,7 +359,7 @@ class MigrateChronicleParticipantStats( tudLastDate = tudStats.second, tudDatesCount = tudStats.third ) - }.values.groupBy { it.studyId } + }.values.groupBy { it.studyEntityKeyId } } // start, end date, count @@ -441,11 +451,11 @@ class MigrateChronicleParticipantStats( return entitySetNameByTemplateName.filter { entitySetIds.keys.contains(it.value) }.mapValues { entitySetIds.getValue(it.value) } } - private fun getParticipantFromNeighborEntity(studyId: UUID, entity: NeighborEntityDetails): Participant { + private fun getParticipantFromNeighborEntity(studyEntityKeyId: UUID, entity: NeighborEntityDetails): Participant { val id = getFirstUUIDOrNull(entity.neighborDetails.get(), OL_ID_FQN) val participantId = getFirstValueOrNull(entity.neighborDetails.get(), PERSON_FQN) - return Participant(studyId, id!!, participantId!!) // hope this force unwrapping doesn't throw NPE + return Participant(studyEntityKeyId, id!!, participantId!!) // hope this force unwrapping doesn't throw NPE } @@ -470,20 +480,22 @@ class MigrateChronicleParticipantStats( return null } - private fun getStudyEntity(studyId: UUID, entity: Map>): Study { + private fun getStudyEntity(studyEntityKeyId: UUID, entity: Map>): Study { val title = getFirstValueOrNull(entity, FULL_NAME_FQN) - val generalStringId = getFirstUUIDOrNull(entity, STRING_ID_FQN) - return Study(studyId, generalStringId!!, title) + val studyId = getFirstUUIDOrNull(entity, STRING_ID_FQN) + return Study(studyEntityKeyId, studyId!!, title) } } private data class Participant( - val studyId: UUID, + val studyEntityKeyId: UUID, val id: UUID, val participantId: String, ) private data class ParticipantStats( + val organizationId: UUID, + val studyEntityKeyId: UUID, val studyId: UUID, val participantId: String, val androidFirstDate: Any?, @@ -495,7 +507,7 @@ private data class ParticipantStats( ) private data class Study( + val studyEntityKeyId: UUID, val studyId: UUID, - val generalStringId: UUID, val title: String? ) \ No newline at end of file From ee4efd50d1245fc1e996e8076b14c638d18535ce Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Thu, 24 Feb 2022 09:18:38 -0800 Subject: [PATCH 27/99] check for duplicates --- .../mechanic/upgrades/MigrateChronicleParticipantStats.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt index b708e8c0..d9b2373a 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt @@ -227,9 +227,9 @@ class MigrateChronicleParticipantStats( // check for duplicate participant ids. for each study, participantIds should be unique - participants.forEach { (studyEntityKeyId, participants) -> - val distinctParticipants = participants.distinct() - val difference = participants - distinctParticipants.toSet() + participants.forEach { (studyEntityKeyId, studyParticipants) -> + val distinctParticipants = studyParticipants.distinct() + val difference = studyParticipants - distinctParticipants.toSet() if (difference.isNotEmpty()) { logger.info("Found duplicate participant ids in ${studies.getValue(studyEntityKeyId)}") } From 0c2dd182a9d67a9705eb39c3a068875e50353522 Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Thu, 24 Feb 2022 09:38:50 -0800 Subject: [PATCH 28/99] use set --- .../upgrades/MigrateChronicleParticipantStats.kt | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt index d9b2373a..7c6d1c17 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt @@ -226,15 +226,6 @@ class MigrateChronicleParticipantStats( logger.info("Participant count by study: ${participants.map { studies.getValue(it.key).title to it.value.size }.toMap()}") - // check for duplicate participant ids. for each study, participantIds should be unique - participants.forEach { (studyEntityKeyId, studyParticipants) -> - val distinctParticipants = studyParticipants.distinct() - val difference = studyParticipants - distinctParticipants.toSet() - if (difference.isNotEmpty()) { - logger.info("Found duplicate participant ids in ${studies.getValue(studyEntityKeyId)}") - } - } - // step 3: neighbor search on participant entity set val participantStats = getParticipantStats( participantEntitySets = participantEntitySets, @@ -302,13 +293,13 @@ class MigrateChronicleParticipantStats( entityKeyIds: Set, principals: Set ) - : Map> { + : Map> { val filter = EntityNeighborsFilter(entityKeyIds, Optional.of(participantEntitySetIds), Optional.empty(), Optional.of(setOf(edgeEntitySetId))) return searchService .executeEntityNeighborSearch(setOf(studiesEntitySetId), PagedNeighborRequest(filter), principals) .neighbors - .mapValues { it.value.map { neighbor -> getParticipantFromNeighborEntity(it.key, neighbor) } } + .mapValues { it.value.map { neighbor -> getParticipantFromNeighborEntity(it.key, neighbor) }.toSet() } } From 9d9fdf2bcd1cf9083d690ff7c52ea34d6b984a17 Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Thu, 24 Feb 2022 09:41:15 -0800 Subject: [PATCH 29/99] change col name --- .../mechanic/upgrades/MigrateChronicleParticipantStats.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt index 7c6d1c17..89b3e134 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt @@ -89,8 +89,8 @@ class MigrateChronicleParticipantStats( // column names private const val ORGANIZATION_IO = "organization_id" - private const val V2_STUDY_ID = "v2_study_id" - private const val V2_STUDY_EKID = "v2_study_ekid" + private const val V2_STUDY_ID = "legacy_study_id" + private const val V2_STUDY_EKID = "legacy_study_ekid" private const val PARTICIPANT_ID = "participant_id" private const val ANDROID_FIRST_DATE = "android_first_date" private const val ANDROID_LAST_DATE = "android_last_date" From bf8f63051315a53f913925e4a520ad8a916850f0 Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Thu, 24 Feb 2022 09:58:14 -0800 Subject: [PATCH 30/99] log stuff --- .../mechanic/upgrades/MigrateChronicleParticipantStats.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt index 89b3e134..96bcac13 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt @@ -222,6 +222,7 @@ class MigrateChronicleParticipantStats( principals = principals, edgeEntitySetId = entitySets.getValue(PARTICIPATED_IN_ES) ) + logger.info("participant entity sets: $participantEntitySets") logger.info("Retrieved ${participants.values.flatten().size} participants") logger.info("Participant count by study: ${participants.map { studies.getValue(it.key).title to it.value.size }.toMap()}") From 1a261b7d1fe69ccf288035f9cd585639e2515cd7 Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Thu, 24 Feb 2022 10:13:36 -0800 Subject: [PATCH 31/99] permission stuff --- .../mechanic/upgrades/MigrateChronicleParticipantStats.kt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt index 96bcac13..7f195b6f 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt @@ -184,6 +184,7 @@ class MigrateChronicleParticipantStats( val orgIdsByAppId = getOrgIdsByAppId().toMutableMap() orgIdsByAppId.getValue(DATA_COLLECTION_APP_ID).add(LEGACY_ORG_ID) + val superUserPrincipals = getChronicleSuperUserPrincipals() (orgIdsByAppId.values.flatten().toSet()).forEach { orgId -> logger.info("---------------------------------------------") @@ -196,7 +197,7 @@ class MigrateChronicleParticipantStats( logger.warn("skipping {} since it doesn't have admin role", orgId) return@forEach } - val principals = principalService.getAllUsersWithPrincipal(adminRoleAclKey).map { it.principal }.toSet() + getChronicleSuperUserPrincipals() + val principals = principalService.getAllUsersWithPrincipal(adminRoleAclKey).map { it.principal }.toSet() + superUserPrincipals val entitySets = getOrgEntitySetNames(orgId) @@ -212,7 +213,7 @@ class MigrateChronicleParticipantStats( val participantEntitySets = when (orgId) { LEGACY_ORG_ID -> getLegacyParticipantEntitySetIds(studies.values.map { it.studyId }.toSet()) else -> setOf(entitySets.getValue(PARTICIPANTS_ES)) - }.filter { authorizationService.checkIfHasPermissions(AclKey(it), principals, EnumSet.of(Permission.READ)) }.toSet() + }.filter { authorizationService.checkIfHasPermissions(AclKey(it), superUserPrincipals, EnumSet.of(Permission.READ)) }.toSet() logger.info("participant entity sets: $participantEntitySets") val participants = getOrgParticipants( From bad2f0eb2f3f6b681d7aa94ddb98b39ff11429f4 Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Thu, 24 Feb 2022 10:26:24 -0800 Subject: [PATCH 32/99] resolve duplicates --- .../MigrateChronicleParticipantStats.kt | 34 ++++++++++++++++--- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt index 7f195b6f..c7c4de38 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt @@ -222,11 +222,22 @@ class MigrateChronicleParticipantStats( entityKeyIds = studies.keys, principals = principals, edgeEntitySetId = entitySets.getValue(PARTICIPATED_IN_ES) - ) + ).toMutableMap() + logger.info("participant entity sets: $participantEntitySets") logger.info("Retrieved ${participants.values.flatten().size} participants") logger.info("Participant count by study: ${participants.map { studies.getValue(it.key).title to it.value.size }.toMap()}") + // check for duplicates + participants.forEach { (studyEntityKeyId, studyParticipants) -> + val unique = studyParticipants.distinct() + val duplicates = studyParticipants - unique.toSet() + + if (duplicates.isNotEmpty()) { + logger.info("Found duplicate participants in study ${studies.getValue(studyEntityKeyId)}: $duplicates") + participants[studyEntityKeyId] = unique + } + } // step 3: neighbor search on participant entity set val participantStats = getParticipantStats( @@ -295,13 +306,13 @@ class MigrateChronicleParticipantStats( entityKeyIds: Set, principals: Set ) - : Map> { + : Map> { val filter = EntityNeighborsFilter(entityKeyIds, Optional.of(participantEntitySetIds), Optional.empty(), Optional.of(setOf(edgeEntitySetId))) return searchService .executeEntityNeighborSearch(setOf(studiesEntitySetId), PagedNeighborRequest(filter), principals) .neighbors - .mapValues { it.value.map { neighbor -> getParticipantFromNeighborEntity(it.key, neighbor) }.toSet() } + .mapValues { it.value.map { neighbor -> getParticipantFromNeighborEntity(it.key, neighbor) } } } @@ -480,11 +491,24 @@ class MigrateChronicleParticipantStats( } } -private data class Participant( +class Participant( val studyEntityKeyId: UUID, val id: UUID, val participantId: String, -) +) { + override fun equals(other: Any?): Boolean { + other as Participant + if (other.participantId == this.participantId && other.studyEntityKeyId == this.studyEntityKeyId) return true + return false + } + + override fun hashCode(): Int { + var result = super.hashCode() + result = 31 * result + studyEntityKeyId.hashCode() + result = 31 * result + participantId.hashCode() + return result + } +} private data class ParticipantStats( val organizationId: UUID, From 32caaca1433fa89c4bb3df2dfdde01ec99afee6c Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Thu, 24 Feb 2022 10:27:24 -0800 Subject: [PATCH 33/99] super user --- .../mechanic/upgrades/MigrateChronicleParticipantStats.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt index c7c4de38..039e548a 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt @@ -44,7 +44,7 @@ class MigrateChronicleParticipantStats( companion object { private val logger = LoggerFactory.getLogger(MigrateChronicleParticipantStats::class.java) - private val chronicleSuperUserIds = setOf("auth0|5ae9026c04eb0b243f1d2bb6", "google-oauth2|113860246540203337319") + private val chronicleSuperUserIds = setOf("auth0|5ae9026c04eb0b243f1d2bb6") private val LEGACY_ORG_ID = UUID.fromString("7349c446-2acc-4d14-b2a9-a13be39cff93") private val DATA_COLLECTION_APP_ID = UUID.fromString("c4e6d8fd-daf9-41e7-8c59-2a12c7ee0857") From 33d5aa247fe1b8dd312648908aacd3fa0a5cb6fa Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Thu, 24 Feb 2022 10:41:15 -0800 Subject: [PATCH 34/99] unnecessary filtering --- .../mechanic/upgrades/MigrateChronicleParticipantStats.kt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt index 039e548a..ffa27f8a 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt @@ -44,7 +44,7 @@ class MigrateChronicleParticipantStats( companion object { private val logger = LoggerFactory.getLogger(MigrateChronicleParticipantStats::class.java) - private val chronicleSuperUserIds = setOf("auth0|5ae9026c04eb0b243f1d2bb6") + private val chronicleSuperUserIds = setOf("auth0|5ae9026c04eb0b243f1d2bb6", "google-oauth2|113860246540203337319") private val LEGACY_ORG_ID = UUID.fromString("7349c446-2acc-4d14-b2a9-a13be39cff93") private val DATA_COLLECTION_APP_ID = UUID.fromString("c4e6d8fd-daf9-41e7-8c59-2a12c7ee0857") @@ -213,7 +213,8 @@ class MigrateChronicleParticipantStats( val participantEntitySets = when (orgId) { LEGACY_ORG_ID -> getLegacyParticipantEntitySetIds(studies.values.map { it.studyId }.toSet()) else -> setOf(entitySets.getValue(PARTICIPANTS_ES)) - }.filter { authorizationService.checkIfHasPermissions(AclKey(it), superUserPrincipals, EnumSet.of(Permission.READ)) }.toSet() + } +// }.filter { authorizationService.checkIfHasPermissions(AclKey(it), superUserPrincipals, EnumSet.of(Permission.READ)) }.toSet() logger.info("participant entity sets: $participantEntitySets") val participants = getOrgParticipants( From 8d545aaddebfabe586f1d8bab5dd4af036a89de1 Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Thu, 24 Feb 2022 11:26:59 -0800 Subject: [PATCH 35/99] super user principal --- .../mechanic/upgrades/MigrateChronicleParticipantStats.kt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt index ffa27f8a..08c921b6 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt @@ -44,7 +44,7 @@ class MigrateChronicleParticipantStats( companion object { private val logger = LoggerFactory.getLogger(MigrateChronicleParticipantStats::class.java) - private val chronicleSuperUserIds = setOf("auth0|5ae9026c04eb0b243f1d2bb6", "google-oauth2|113860246540203337319") + private val chronicleSuperUserIds = setOf("auth0|5ae9026c04eb0b243f1d2bb6") private val LEGACY_ORG_ID = UUID.fromString("7349c446-2acc-4d14-b2a9-a13be39cff93") private val DATA_COLLECTION_APP_ID = UUID.fromString("c4e6d8fd-daf9-41e7-8c59-2a12c7ee0857") @@ -197,7 +197,7 @@ class MigrateChronicleParticipantStats( logger.warn("skipping {} since it doesn't have admin role", orgId) return@forEach } - val principals = principalService.getAllUsersWithPrincipal(adminRoleAclKey).map { it.principal }.toSet() + superUserPrincipals +// val principals = principalService.getAllUsersWithPrincipal(adminRoleAclKey).map { it.principal }.toSet() + superUserPrincipals val entitySets = getOrgEntitySetNames(orgId) @@ -221,7 +221,7 @@ class MigrateChronicleParticipantStats( participantEntitySetIds = participantEntitySets, studiesEntitySetId = entitySets.getValue(STUDIES_ES), entityKeyIds = studies.keys, - principals = principals, + principals = superUserPrincipals, edgeEntitySetId = entitySets.getValue(PARTICIPATED_IN_ES) ).toMutableMap() @@ -246,7 +246,7 @@ class MigrateChronicleParticipantStats( entitySetIds = entitySets, orgIdsByAppId = orgIdsByAppId, orgId = orgId, - principals = principals, + principals = superUserPrincipals, participantById = participants.values.flatten().associateBy { it.id }, studies = studies ) From 21c3830d3189053f363cd64f0fcb0adf020ca372 Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Fri, 25 Feb 2022 10:29:01 -0800 Subject: [PATCH 36/99] migrate org settings to studies --- .../mechanic/pods/MechanicUpgradePod.kt | 11 + .../upgrades/MigrateOrgSettingsToStudies.kt | 214 ++++++++++++++++++ 2 files changed, 225 insertions(+) create mode 100644 src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateOrgSettingsToStudies.kt diff --git a/src/main/kotlin/com/openlattice/mechanic/pods/MechanicUpgradePod.kt b/src/main/kotlin/com/openlattice/mechanic/pods/MechanicUpgradePod.kt index 3767ea48..8ba056f3 100644 --- a/src/main/kotlin/com/openlattice/mechanic/pods/MechanicUpgradePod.kt +++ b/src/main/kotlin/com/openlattice/mechanic/pods/MechanicUpgradePod.kt @@ -72,6 +72,7 @@ import com.openlattice.linking.graph.PostgresLinkingQueryService import com.openlattice.mechanic.MechanicCli.Companion.UPGRADE import com.openlattice.mechanic.Toolbox import com.openlattice.mechanic.upgrades.DeleteOrgMetadataEntitySets +import com.openlattice.mechanic.upgrades.MigrateOrgSettingsToStudies import com.openlattice.mechanic.upgrades.V3StudyMigrationUpgrade import com.openlattice.organizations.roles.HazelcastPrincipalService import com.openlattice.organizations.roles.SecurePrincipalsManager @@ -379,6 +380,16 @@ class MechanicUpgradePod { ) } + @Bean + fun migrateOrgSettingsToStudies(): MigrateOrgSettingsToStudies { + return MigrateOrgSettingsToStudies( + toolbox, + entitySetService(), + rhizomeConfiguration, + dataQueryService() + ) + } + @PostConstruct fun post() { Principals.init(principalService(), hazelcastInstance) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateOrgSettingsToStudies.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateOrgSettingsToStudies.kt new file mode 100644 index 00000000..173c6432 --- /dev/null +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateOrgSettingsToStudies.kt @@ -0,0 +1,214 @@ +package com.openlattice.mechanic.upgrades + +import com.geekbeast.mappers.mappers.ObjectMappers +import com.geekbeast.rhizome.configuration.RhizomeConfiguration +import com.openlattice.data.storage.postgres.PostgresEntityDataQueryService +import com.openlattice.datastore.services.EntitySetManager +import com.openlattice.edm.EdmConstants +import com.openlattice.hazelcast.HazelcastMap +import com.openlattice.mechanic.Toolbox +import com.zaxxer.hikari.HikariConfig +import com.zaxxer.hikari.HikariDataSource +import org.apache.olingo.commons.api.edm.FullQualifiedName +import org.slf4j.LoggerFactory +import java.util.* + +/** + * @author alfoncenzioka <alfonce@openlattice.com> + */ +class MigrateOrgSettingsToStudies( + val toolbox: Toolbox, + private val entitySetsManager: EntitySetManager, + private val rhizomeConfiguration: RhizomeConfiguration, + private val dataQueryService: PostgresEntityDataQueryService + +) : Upgrade { + private val entitySetIdsByName: Map = HazelcastMap.ENTITY_SETS.getMap(toolbox.hazelcast).values.associate { it.name to it.id } + + companion object { + private val logger = LoggerFactory.getLogger(MigrateOrgSettingsToStudies::class.java) + private val mapper = ObjectMappers.getJsonMapper() + + // app ids + private val DATA_COLLECTION_APP_ID = UUID.fromString("c4e6d8fd-daf9-41e7-8c59-2a12c7ee0857") + private val SURVEY_APP_ID = UUID.fromString("bb44218b-515a-4314-b955-df2c991b2575") + private val CHRONICLE_APP_ID = UUID.fromString("82e5504b-4dca-4600-a321-fa8e00e3b788") + + private val LEGACY_ORG_ID = UUID.fromString("7349c446-2acc-4d14-b2a9-a13be39cff93") + private val RICE_UNIVERSITY_ORG_ID = UUID.fromString("a77a8f87-9e3f-4ae1-bfb1-c5c72f194fa8") + + private const val LEGACY_STUDY_ENTITY_SET = "chronicle_study" + + private val OL_ID_FQN = EdmConstants.ID_FQN + private val STRING_ID_FQN = FullQualifiedName("general.stringid") + + private val appIdToComponentMapping = mapOf( + DATA_COLLECTION_APP_ID to AppComponents.CHRONICLE_DATA_COLLECTION, + SURVEY_APP_ID to AppComponents.CHRONICLE_SURVEYS, + CHRONICLE_APP_ID to AppComponents.CHRONICLE + ) + + // settings keys + private const val APP_FREQUENCY_SETTING = "appUsageFrequency" + private const val COMPONENTS_SETTING = "components" + + // columns to insert + private const val LEGACY_STUDY_ID = "v2_study_id" + private const val LEGACY_STUDY_EK_ID = "v2_study_ekid" + private const val SETTINGS = "settings" + + private const val STUDY_SETTINGS_TABLE = "study_settings" + + private val CREATE_TABLE_QUERY = """ + CREATE TABLE IF NOT EXISTS $STUDY_SETTINGS_TABLE( + $LEGACY_STUDY_EK_ID uuid NOT NULL, + $LEGACY_STUDY_ID uuid NOT NULL, + $SETTINGS jsonb, + PRIMARY KEY ($LEGACY_STUDY_EK_ID) + ) + """.trimIndent() + + /** + * PreparedStatement bind order + * 1) studyEntityKeyId + * 2) studyId + * 3) settings + */ + private val INSERT_INTO_SETTINGS_TABLE_QUERY = """ + INSERT INTO $STUDY_SETTINGS_TABLE ($LEGACY_STUDY_EK_ID, $LEGACY_STUDY_ID, $SETTINGS) VALUES (?, ?, ?::jsonb) + """.trimIndent() + } + + init { + getDataSource().connection.createStatement().use { stmt -> + stmt.execute(CREATE_TABLE_QUERY) + } + } + + override fun upgrade(): Boolean { + val appConfigs = HazelcastMap.APP_CONFIGS.getMap(toolbox.hazelcast) + + val appIdsByOrganizationId: MutableMap> = appConfigs.keys + .filter { it.appId == DATA_COLLECTION_APP_ID || it.appId == SURVEY_APP_ID || it.appId == CHRONICLE_APP_ID } + .groupBy { it.organizationId } + .mapValues { (_, configs) -> configs.map { it.appId }.toSet() } + .toMutableMap() + + // legacy org id should have all app components + appIdsByOrganizationId[LEGACY_ORG_ID] = appIdToComponentMapping.keys + + val studiesByOrgId: Map> = getStudiesByOrgId(appIdsByOrganizationId.keys, appIdsByOrganizationId) + logger.info("Entities to write by org: ${studiesByOrgId.mapValues { it.value.size }}") + + val rowsWritten = writeEntitiesToTable(studiesByOrgId) + logger.info("Wrote $rowsWritten study setting entities to table") + return true + } + + private fun writeEntitiesToTable(entities: Map>): Int { + return getDataSource().connection.use { connection -> + try { + val wc = connection.prepareStatement(INSERT_INTO_SETTINGS_TABLE_QUERY).use { ps -> + entities.values.flatten().forEach { + ps.setObject(1, it.studyEntityKeyId) + ps.setObject(2, it.studyId) + ps.setString(3, mapper.writeValueAsString(it.settings)) + ps.addBatch() + } + ps.executeBatch().sum() + } + return@use wc + } catch (ex: Exception) { + throw ex + } + } + } + + private fun getDataSource(): HikariDataSource { + val (hikariConfiguration) = rhizomeConfiguration.datasourceConfigurations["chronicle"]!! + val hc = HikariConfig(hikariConfiguration) + return HikariDataSource(hc) + } + + private fun getStudiesByOrgId(orgIds: Set, appIdsByOrganizationId: Map>): Map> { + val studyIdEntitySetIdByOrgId = getEntitySetIds(orgIds) + + return orgIds.associateWith { orgId -> + getStudyEntities( + orgId, + appIdsByOrganizationId.getValue(orgId), + studyIdEntitySetIdByOrgId.getValue(orgId) + ) + } + } + + private fun getStudyEntities(orgId: UUID, appIds: Set, entitySetId: UUID): List { + return dataQueryService.getEntitiesWithPropertyTypeFqns( + mapOf(entitySetId to Optional.empty()), + entitySetsManager.getPropertyTypesOfEntitySets(setOf(entitySetId)), + mapOf(), + setOf(), + Optional.empty(), + false + ).values.map { getStudyEntity(it, appIds, orgId) }.toList() + } + + private fun getStudyEntity(entity: Map>, appIds: Set, orgId: UUID): Study { + val studyId = getFirstUUIDOrNull(entity, STRING_ID_FQN) + val studyEntityKeyId = getFirstUUIDOrNull(entity, OL_ID_FQN) + + val settings: MutableMap = mutableMapOf() + settings[COMPONENTS_SETTING] = appIds.map { appIdToComponentMapping.getValue(it) }.toSet() + settings[APP_FREQUENCY_SETTING] = if (orgId == RICE_UNIVERSITY_ORG_ID) AppUsageFrequency.HOURLY else AppUsageFrequency.DAILY + + return Study( + studyEntityKeyId = studyEntityKeyId!!, + studyId = studyId!!, + settings = settings + ) + } + + private fun getEntitySetIds(orgIds: Set): Map { + return orgIds.associateWith { orgId -> + when (orgId) { + LEGACY_ORG_ID -> LEGACY_STUDY_ENTITY_SET + else -> "chronicle_${orgId.toString().replace("-", "")}_studies" + } + }.mapValues { entitySetIdsByName.getValue(it.value) } + } + + private fun getFirstUUIDOrNull(entity: Map>, fqn: FullQualifiedName): UUID? { + return when (val string = getFirstValueOrNull(entity, fqn)) { + null -> null + else -> UUID.fromString(string) + } + } + + private fun getFirstValueOrNull(entity: Map>, fqn: FullQualifiedName): String? { + entity[fqn]?.iterator()?.let { + if (it.hasNext()) return it.next().toString() + } + return null + } + + override fun getSupportedVersion(): Long { + return Version.V2021_07_23.value + } +} + +private data class Study( + val studyId: UUID, + val studyEntityKeyId: UUID, + val settings: Map +) + +private enum class AppComponents { + CHRONICLE, + CHRONICLE_DATA_COLLECTION, + CHRONICLE_SURVEYS +} + +private enum class AppUsageFrequency { + DAILY, + HOURLY +} \ No newline at end of file From 1db19373ee2e64212affe17957ebd857607759a9 Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Fri, 25 Feb 2022 10:59:08 -0800 Subject: [PATCH 37/99] use chronicle datasource --- .../mechanic/upgrades/MigrateChronicleParticipantStats.kt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt index 08c921b6..b6dd7bb5 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt @@ -173,9 +173,8 @@ class MigrateChronicleParticipantStats( return Version.V2021_07_23.value } - private fun getHikariDataSource(): HikariDataSource { - val (hikariConfiguration) = rhizomeConfiguration.datasourceConfigurations["alpr"]!! + val (hikariConfiguration) = rhizomeConfiguration.datasourceConfigurations["chronicle"]!! val hc = HikariConfig(hikariConfiguration) return HikariDataSource(hc) } From ee3ef19afd61d95c6262abbc3b85bbd7d8cd819f Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Fri, 25 Feb 2022 11:03:59 -0800 Subject: [PATCH 38/99] remove unused --- .../MigrateChronicleParticipantStats.kt | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt index b6dd7bb5..b49efe23 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt @@ -44,7 +44,7 @@ class MigrateChronicleParticipantStats( companion object { private val logger = LoggerFactory.getLogger(MigrateChronicleParticipantStats::class.java) - private val chronicleSuperUserIds = setOf("auth0|5ae9026c04eb0b243f1d2bb6") + private const val SUPER_USER_PRINCIPAL_ID = "auth0|5ae9026c04eb0b243f1d2bb6" private val LEGACY_ORG_ID = UUID.fromString("7349c446-2acc-4d14-b2a9-a13be39cff93") private val DATA_COLLECTION_APP_ID = UUID.fromString("c4e6d8fd-daf9-41e7-8c59-2a12c7ee0857") @@ -185,22 +185,21 @@ class MigrateChronicleParticipantStats( orgIdsByAppId.getValue(DATA_COLLECTION_APP_ID).add(LEGACY_ORG_ID) val superUserPrincipals = getChronicleSuperUserPrincipals() + logger.info("Using principals: $superUserPrincipals") + (orgIdsByAppId.values.flatten().toSet()).forEach { orgId -> logger.info("---------------------------------------------") logger.info("Retrieving entities in organization $orgId") logger.info("----------------------------------------------") - // get principals val adminRoleAclKey = organizations[orgId]?.adminRoleAclKey if (adminRoleAclKey == null) { logger.warn("skipping {} since it doesn't have admin role", orgId) return@forEach } -// val principals = principalService.getAllUsersWithPrincipal(adminRoleAclKey).map { it.principal }.toSet() + superUserPrincipals - - val entitySets = getOrgEntitySetNames(orgId) - + // step 1: get studies in org: {studyEntityKeyId -> study} + val entitySets = getOrgEntitySetNames(orgId) val studies: Map = getOrgStudies(entitySetId = entitySets.getValue(STUDIES_ES)) if (studies.isEmpty()) { logger.info("organization $orgId has no studies. Skipping") @@ -419,10 +418,8 @@ class MigrateChronicleParticipantStats( } private fun getChronicleSuperUserPrincipals(): Set { - return chronicleSuperUserIds - .map { principalService.getSecurablePrincipal(it) } - .map { principalService.getAllPrincipals(it).map { principal -> principal.principal } } - .flatten().toSet() + val securablePrincipal = principalService.getSecurablePrincipal(SUPER_USER_PRINCIPAL_ID) + return principalService.getAllPrincipals(securablePrincipal).map { it.principal }.toSet() } private fun getLegacyParticipantEntitySetIds(studyIds: Set): Set { From 196364c48ede0f061a041fce5d75ff414f87dae4 Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Sat, 26 Feb 2022 22:27:11 -0800 Subject: [PATCH 39/99] remove unnecessary --- .../mechanic/upgrades/MigrateChronicleParticipantStats.kt | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt index b49efe23..1311288d 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt @@ -212,7 +212,6 @@ class MigrateChronicleParticipantStats( LEGACY_ORG_ID -> getLegacyParticipantEntitySetIds(studies.values.map { it.studyId }.toSet()) else -> setOf(entitySets.getValue(PARTICIPANTS_ES)) } -// }.filter { authorizationService.checkIfHasPermissions(AclKey(it), superUserPrincipals, EnumSet.of(Permission.READ)) }.toSet() logger.info("participant entity sets: $participantEntitySets") val participants = getOrgParticipants( @@ -223,7 +222,6 @@ class MigrateChronicleParticipantStats( edgeEntitySetId = entitySets.getValue(PARTICIPATED_IN_ES) ).toMutableMap() - logger.info("participant entity sets: $participantEntitySets") logger.info("Retrieved ${participants.values.flatten().size} participants") logger.info("Participant count by study: ${participants.map { studies.getValue(it.key).title to it.value.size }.toMap()}") From 2e6573600d3592898963f18cc4545e73c2e0d2bb Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Sat, 26 Feb 2022 22:57:37 -0800 Subject: [PATCH 40/99] use principal type user --- .../mechanic/upgrades/MigrateChronicleParticipantStats.kt | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt index 1311288d..ec9214e5 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt @@ -1,10 +1,7 @@ package com.openlattice.mechanic.upgrades import com.geekbeast.rhizome.configuration.RhizomeConfiguration -import com.openlattice.authorization.AclKey -import com.openlattice.authorization.AuthorizationManager -import com.openlattice.authorization.Permission -import com.openlattice.authorization.Principal +import com.openlattice.authorization.* import com.openlattice.data.requests.NeighborEntityDetails import com.openlattice.data.storage.postgres.PostgresEntityDataQueryService import com.openlattice.datastore.services.EntitySetManager @@ -417,7 +414,7 @@ class MigrateChronicleParticipantStats( private fun getChronicleSuperUserPrincipals(): Set { val securablePrincipal = principalService.getSecurablePrincipal(SUPER_USER_PRINCIPAL_ID) - return principalService.getAllPrincipals(securablePrincipal).map { it.principal }.toSet() + return principalService.getAllPrincipals(securablePrincipal).map { it.principal }.toSet() + Principal(PrincipalType.USER, SUPER_USER_PRINCIPAL_ID) } private fun getLegacyParticipantEntitySetIds(studyIds: Set): Set { From f8967e6fc3cc5bff0869b062c6a1f230ffbbe575 Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Sat, 26 Feb 2022 23:13:53 -0800 Subject: [PATCH 41/99] optional might not exist --- .../mechanic/upgrades/MigrateChronicleParticipantStats.kt | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt index ec9214e5..c15d5020 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt @@ -373,9 +373,12 @@ class MigrateChronicleParticipantStats( val dateTimeEndValues = getOffsetDateTimesFromNeighborEntities(neighbors, DATE_TIME_END_FQN) val datesRecorded = getOffsetDateTimesFromNeighborEntities(neighbors, RECORDED_DATE_FQN) + val firstDate = dateTimeStartValues.stream().min(OffsetDateTime::compareTo) + val lastDate = dateTimeEndValues.stream().max(OffsetDateTime::compareTo) + return Triple( - first = dateTimeStartValues.stream().min(OffsetDateTime::compareTo).get(), - second = dateTimeEndValues.stream().max(OffsetDateTime::compareTo).get(), + first = if (firstDate.isEmpty) null else firstDate.get(), + second = if (lastDate.isEmpty) null else lastDate.get(), third = datesRecorded.map { it.toLocalDate() }.toSet().size // unique dates ) } From 0c23f652fea2dc93767d3a867acdd1c3088c1d0e Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Sat, 26 Feb 2022 23:15:31 -0800 Subject: [PATCH 42/99] clean up stuff --- .../upgrades/MigrateChronicleParticipantStats.kt | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt index c15d5020..10835bde 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt @@ -222,17 +222,6 @@ class MigrateChronicleParticipantStats( logger.info("Retrieved ${participants.values.flatten().size} participants") logger.info("Participant count by study: ${participants.map { studies.getValue(it.key).title to it.value.size }.toMap()}") - // check for duplicates - participants.forEach { (studyEntityKeyId, studyParticipants) -> - val unique = studyParticipants.distinct() - val duplicates = studyParticipants - unique.toSet() - - if (duplicates.isNotEmpty()) { - logger.info("Found duplicate participants in study ${studies.getValue(studyEntityKeyId)}: $duplicates") - participants[studyEntityKeyId] = unique - } - } - // step 3: neighbor search on participant entity set val participantStats = getParticipantStats( participantEntitySets = participantEntitySets, @@ -300,13 +289,13 @@ class MigrateChronicleParticipantStats( entityKeyIds: Set, principals: Set ) - : Map> { + : Map> { val filter = EntityNeighborsFilter(entityKeyIds, Optional.of(participantEntitySetIds), Optional.empty(), Optional.of(setOf(edgeEntitySetId))) return searchService .executeEntityNeighborSearch(setOf(studiesEntitySetId), PagedNeighborRequest(filter), principals) .neighbors - .mapValues { it.value.map { neighbor -> getParticipantFromNeighborEntity(it.key, neighbor) } } + .mapValues { it.value.map { neighbor -> getParticipantFromNeighborEntity(it.key, neighbor) }.toSet() } } From 4f63aeff91b4c5d275477211ccc59ac8f85c709d Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Sat, 26 Feb 2022 23:24:24 -0800 Subject: [PATCH 43/99] fix sql --- .../MigrateChronicleParticipantStats.kt | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt index 10835bde..b519ea94 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt @@ -148,6 +148,7 @@ class MigrateChronicleParticipantStats( */ private val INSERT_PARTICIPANT_STATS_SQL = """ INSERT INTO participant_stats ($PARTICIPANT_STATS_COLS) values ($PARTICIPANT_STATS_PARAMS) + ON CONFLICT DO NOTHING """.trimIndent() } @@ -475,24 +476,11 @@ class MigrateChronicleParticipantStats( } } -class Participant( +data class Participant( val studyEntityKeyId: UUID, val id: UUID, val participantId: String, -) { - override fun equals(other: Any?): Boolean { - other as Participant - if (other.participantId == this.participantId && other.studyEntityKeyId == this.studyEntityKeyId) return true - return false - } - - override fun hashCode(): Int { - var result = super.hashCode() - result = 31 * result + studyEntityKeyId.hashCode() - result = 31 * result + participantId.hashCode() - return result - } -} +) private data class ParticipantStats( val organizationId: UUID, From 6810be1df20e645a0b7786c5057284ea65ddeef5 Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Sat, 26 Feb 2022 23:44:44 -0800 Subject: [PATCH 44/99] fix sql --- .../openlattice/mechanic/upgrades/MigrateOrgSettingsToStudies.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateOrgSettingsToStudies.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateOrgSettingsToStudies.kt index 173c6432..974a7679 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateOrgSettingsToStudies.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateOrgSettingsToStudies.kt @@ -76,6 +76,7 @@ class MigrateOrgSettingsToStudies( */ private val INSERT_INTO_SETTINGS_TABLE_QUERY = """ INSERT INTO $STUDY_SETTINGS_TABLE ($LEGACY_STUDY_EK_ID, $LEGACY_STUDY_ID, $SETTINGS) VALUES (?, ?, ?::jsonb) + ON CONFLICT DO NOTHING """.trimIndent() } From 17b59e3a515d25ed461ce83a53dfa2dd9ba71a27 Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Sat, 26 Feb 2022 23:56:38 -0800 Subject: [PATCH 45/99] study id might be null --- .../mechanic/upgrades/MigrateOrgSettingsToStudies.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateOrgSettingsToStudies.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateOrgSettingsToStudies.kt index 974a7679..70cbd117 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateOrgSettingsToStudies.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateOrgSettingsToStudies.kt @@ -151,7 +151,7 @@ class MigrateOrgSettingsToStudies( setOf(), Optional.empty(), false - ).values.map { getStudyEntity(it, appIds, orgId) }.toList() + ).values.filter { getFirstUUIDOrNull(it, STRING_ID_FQN) != null } .map { getStudyEntity(it, appIds, orgId) }.toList() } private fun getStudyEntity(entity: Map>, appIds: Set, orgId: UUID): Study { From 5b169f54e84fb2e4c8cc47fdb4b68d17f8ee1d75 Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Sun, 27 Feb 2022 07:13:54 -0800 Subject: [PATCH 46/99] store legacy study id --- .../upgrades/V3AppUsageSurveyMigration.kt | 58 +++++++++---------- 1 file changed, 28 insertions(+), 30 deletions(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/V3AppUsageSurveyMigration.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/V3AppUsageSurveyMigration.kt index 40604d2d..ad31f829 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/V3AppUsageSurveyMigration.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/V3AppUsageSurveyMigration.kt @@ -12,9 +12,7 @@ import com.openlattice.edm.EdmConstants.Companion.LAST_WRITE_FQN import com.openlattice.graph.PagedNeighborRequest import com.openlattice.hazelcast.HazelcastMap import com.openlattice.mechanic.Toolbox -import com.openlattice.organization.PERMISSION import com.openlattice.organizations.roles.SecurePrincipalsManager -import com.openlattice.postgres.PostgresColumn.PERMISSIONS import com.openlattice.search.SearchService import com.openlattice.search.requests.EntityNeighborsFilter import com.zaxxer.hikari.HikariConfig @@ -42,12 +40,10 @@ class V3AppUsageSurveyMigration( private val appConfigs = HazelcastMap.APP_CONFIGS.getMap(toolbox.hazelcast) private val entitySetIds = HazelcastMap.ENTITY_SETS.getMap(toolbox.hazelcast).values.associateBy { it.name } - private val allStudyIdsByParticipantEKID: MutableMap = mutableMapOf() - private val allParticipantIdsByEKID: MutableMap = mutableMapOf() - private val allSurveyDataByParticipantEKID: MutableMap>>> = mutableMapOf() - - //TODO: replace empty strings with chronicle super user ids (auth0 and google-oauth2) when running migration - private val chronicleSuperUserIds = setOf("auth0|5ae9026c04eb0b243f1d2bb6", "google-oauth2|113860246540203337319") + private val studyEKIDByParticipantEKID: MutableMap = mutableMapOf() + private val participantIdByEKID: MutableMap = mutableMapOf() + private val surveyDataByParticipantEKID: MutableMap>>> = mutableMapOf() + private val studyIdByStudyEKID: MutableMap = mutableMapOf() companion object { private val DATA_COLLECTION_APP_ID = UUID.fromString("c4e6d8fd-daf9-41e7-8c59-2a12c7ee0857") @@ -85,7 +81,7 @@ class V3AppUsageSurveyMigration( USER_FQN to "users" ) - private val column_names = listOf("study_id", "participant_id") + FQN_TO_COLUMNS.values.toList() + private val column_names = listOf("v2_study_id", "participant_id") + FQN_TO_COLUMNS.values.toList() private val APP_USAGE_SURVEY_COLUMNS = column_names.joinToString(",") { it } private val APP_USAGE_SURVEY_PARAMS = column_names.joinToString(",") { "?" } @@ -109,7 +105,7 @@ class V3AppUsageSurveyMigration( private val CREATE_APP_USAGE_SURVEY_TABLE_SQL = """ CREATE TABLE IF NOT EXISTS $TABLE_NAME( - study_id uuid NOT NULL, + v2_study_id uuid NOT NULL, participant_id text NOT NULL, submission_date timestamp with time zone NOT NULL, application_label text, @@ -138,11 +134,12 @@ class V3AppUsageSurveyMigration( try { val written = hds.connection.use { connection -> connection.prepareStatement(INSERT_INTO_APP_USAGE_SQL).use { ps -> - allSurveyDataByParticipantEKID.forEach { (key, data) -> - val studyId = allStudyIdsByParticipantEKID[key] - val participantId = allParticipantIdsByEKID[key] + surveyDataByParticipantEKID.forEach { (key, data) -> + val studyEKID = studyEKIDByParticipantEKID[key] + val studyId = studyIdByStudyEKID[studyEKID] + val participantId = participantIdByEKID[key] - if (studyId == null || participantId == null) { + if (studyEKID == null || participantId == null || studyId == null) { logger.warn("Skipping migration for participant $key. failed to retrieve associated studyId or participantId") return@forEach } @@ -188,7 +185,7 @@ class V3AppUsageSurveyMigration( } } - logger.info("wrote {} entities to db. data query service returned {} entities", written, allSurveyDataByParticipantEKID.values.flatten().size) + logger.info("wrote {} entities to db. data query service returned {} entities", written, surveyDataByParticipantEKID.values.flatten().size) return true } catch (ex: Exception) { logger.error("error migrating app usage survey data to v3", ex) @@ -239,44 +236,45 @@ class V3AppUsageSurveyMigration( logger.info("found {} studies in org {}", studies.size, orgId) val studyEntityKeyIds = studies.keys - val studyIds = studies.map { getFirstUUIDOrNull(it.value, STRING_ID_FQN) }.filterNotNull().toSet() + val studyIdByStudyEKID = studies.mapValues { getFirstUUIDOrNull(it.value, STRING_ID_FQN)!! } + val studyIds = studyIdByStudyEKID.values.toSet() val participantEntitySetIds = when (orgId) { LEGACY_ORG_ID -> getLegacyParticipantEntitySetIds(studyIds) else -> setOf(participantsEntitySetId!!) - }.filter { authorizationService.checkIfHasPermissions(AclKey(it), principals, EnumSet.of(Permission.READ)) }.toSet() + } val participants = getStudyParticipants(studiesEntitySetId, studyEntityKeyIds, participantEntitySetIds, participatedInEntitySetId, principals) - val participantsByStudyId = participants.mapValues { (_, neighbor) -> neighbor.map { it.neighborId.get() }.toSet() } - logger.info("org {} participant count by study {}", orgId, participantsByStudyId.mapValues { it.value.size }) + val participantsByStudyEKID = participants.mapValues { (_, neighbor) -> neighbor.map { it.neighborId.get() }.toSet() } + logger.info("org {} participant count by study {}", orgId, participantsByStudyEKID.mapValues { it.value.size }) val participantIdByEKID = participants.values .flatten() .associate { getFirstUUIDOrNull(it.neighborDetails.get(), OL_ID_FQN)!! to getFirstValueOrNull(it.neighborDetails.get(), PERSON_FQN) } - val studyIdByParticipantEKID = participantsByStudyId + val studyEKIDByParticipantEKID = participantsByStudyEKID .map { (studyId, participants) -> participants.associateWith { studyId } } .flatMap { it.asSequence() }.associate { it.key to it.value } - val surveysDataByParticipant = getSurveysDataByParticipant( - principals, participantsByStudyId.values.flatten().toSet(), participantEntitySetIds, usedByEntitySetId, userAppsEntitySetId) + val surveyDataByParticipantEKID = getSurveysDataByParticipant( + principals, participantsByStudyEKID.values.flatten().toSet(), participantEntitySetIds, usedByEntitySetId, userAppsEntitySetId) - logger.info("found {} survey entities in org {}", surveysDataByParticipant.values.flatten().size, orgId) + logger.info("found {} survey entities in org {}", surveyDataByParticipantEKID.values.flatten().size, orgId) - allStudyIdsByParticipantEKID += studyIdByParticipantEKID - allParticipantIdsByEKID += participantIdByEKID - allSurveyDataByParticipantEKID += surveysDataByParticipant + this.studyEKIDByParticipantEKID += studyEKIDByParticipantEKID + this.participantIdByEKID += participantIdByEKID + this.surveyDataByParticipantEKID += surveyDataByParticipantEKID + this.studyIdByStudyEKID += studyIdByStudyEKID } } private fun getChronicleSuperUserPrincipals(): Set { - return chronicleSuperUserIds - .map { principalService.getSecurablePrincipal(it)} - .map { principalService.getAllPrincipals(it).map { principal -> principal.principal } } - .flatten().toSet() + val userId = "auth0|5ae9026c04eb0b243f1d2bb6" + val securablePrincipal = principalService.getSecurablePrincipal(userId) + return principalService.getAllPrincipals(securablePrincipal).map { it.principal }.toSet() + Principal(PrincipalType.USER, userId) } private fun filterInvalidStudies(entities: Map>>): Map>> { From 6d22998cb66a4dfb4841c4b91830d1d0154db79e Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Sun, 27 Feb 2022 07:21:17 -0800 Subject: [PATCH 47/99] write study ekid --- .../upgrades/V3AppUsageSurveyMigration.kt | 60 ++++++++++++------- 1 file changed, 37 insertions(+), 23 deletions(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/V3AppUsageSurveyMigration.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/V3AppUsageSurveyMigration.kt index ad31f829..76a56d60 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/V3AppUsageSurveyMigration.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/V3AppUsageSurveyMigration.kt @@ -72,16 +72,27 @@ class V3AppUsageSurveyMigration( private val TIMEZONE_FQN = FullQualifiedName("ol.timezone") private val TITLE_FQN = FullQualifiedName("ol.title") + // table column names + private const val SUBMISSION_DATE = "submission_date" + private const val APPLICATION_LABEL = "application_label" + private const val APP_PACKAGE_NAME = "app_package_name" + private const val EVENT_TIMESTAMP = "event_timestamp" + private const val TIMEZONE = "timezone" + private const val USERS = "users" + private const val V2_STUDY_ID = "v2_study_id" + private const val PARTICIPANT_ID = "participant_id" + private const val V2_STUDY_EKID = "v2_study_ekid" + private val FQN_TO_COLUMNS = mapOf( - LAST_WRITE_FQN to "submission_date", - TITLE_FQN to "application_label", - FULL_NAME_FQN to "app_package_name", - DATETIME_FQN to "event_timestamp", - TIMEZONE_FQN to "timezone", - USER_FQN to "users" + LAST_WRITE_FQN to SUBMISSION_DATE, + TITLE_FQN to APPLICATION_LABEL, + FULL_NAME_FQN to APP_PACKAGE_NAME, + DATETIME_FQN to EVENT_TIMESTAMP, + TIMEZONE_FQN to TIMEZONE, + USER_FQN to USERS ) - private val column_names = listOf("v2_study_id", "participant_id") + FQN_TO_COLUMNS.values.toList() + private val column_names = listOf(V2_STUDY_EKID, V2_STUDY_ID, PARTICIPANT_ID) + FQN_TO_COLUMNS.values.toList() private val APP_USAGE_SURVEY_COLUMNS = column_names.joinToString(",") { it } private val APP_USAGE_SURVEY_PARAMS = column_names.joinToString(",") { "?" } @@ -89,14 +100,15 @@ class V3AppUsageSurveyMigration( /** * PreparedStatement bind order - * 1) studyId - * 2) participantId - * 3) submissionDate, - * 4) appLabel - * 5) packageName - * 6) timestamp - * 7) timezone - * 8) users + * 1) studyEKID + * 2) studyId + * 3) participantId + * 4) submissionDate, + * 5) appLabel + * 6) packageName + * 7) timestamp + * 8) timezone + * 9) users */ private val INSERT_INTO_APP_USAGE_SQL = """ INSERT INTO app_usage_survey($APP_USAGE_SURVEY_COLUMNS) values ($APP_USAGE_SURVEY_PARAMS) @@ -144,8 +156,9 @@ class V3AppUsageSurveyMigration( return@forEach } - ps.setObject(1, studyId) - ps.setString(2, participantId) + ps.setObject(1, studyEKID) + ps.setObject(2, studyId) + ps.setString(3, participantId) data.forEach data@{ entity -> val users = entity.getOrDefault(USER_FQN, listOf()) @@ -171,12 +184,13 @@ class V3AppUsageSurveyMigration( val timestamps = entity[DATETIME_FQN]?.map { it.toString() }?.toSet() ?: setOf() timestamps.forEach { timestamp -> - ps.setObject(3, OffsetDateTime.parse(submissionDate)) - ps.setString(4, applicationLabel) - ps.setString(5, appPackageName) - ps.setObject(6, OffsetDateTime.parse(timestamp)) - ps.setString(7, timezone) - ps.setArray(8, PostgresArrays.createTextArray(connection, users)) + var index = 3 + ps.setObject(++index, OffsetDateTime.parse(submissionDate)) + ps.setString(++index, applicationLabel) + ps.setString(++index, appPackageName) + ps.setObject(++index, OffsetDateTime.parse(timestamp)) + ps.setString(++index, timezone) + ps.setArray(++index, PostgresArrays.createTextArray(connection, users)) ps.addBatch() } } From aa0074b7743364cf860f578f97a66614b47f1452 Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Sun, 27 Feb 2022 08:12:51 -0800 Subject: [PATCH 48/99] fix create table sql --- .../upgrades/V3AppUsageSurveyMigration.kt | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/V3AppUsageSurveyMigration.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/V3AppUsageSurveyMigration.kt index 76a56d60..a0ca352b 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/V3AppUsageSurveyMigration.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/V3AppUsageSurveyMigration.kt @@ -117,15 +117,16 @@ class V3AppUsageSurveyMigration( private val CREATE_APP_USAGE_SURVEY_TABLE_SQL = """ CREATE TABLE IF NOT EXISTS $TABLE_NAME( - v2_study_id uuid NOT NULL, - participant_id text NOT NULL, - submission_date timestamp with time zone NOT NULL, - application_label text, - app_package_name text NOT NULL, - event_timestamp timestamp with time zone NOT NULL, - timezone text, - users text[], - PRIMARY KEY(app_package_name, event_timestamp) + $V2_STUDY_EKID uuid NOT NULL, + $V2_STUDY_ID uuid NOT NULL, + $PARTICIPANT_ID text NOT NULL, + $SUBMISSION_DATE timestamp with time zone NOT NULL, + $APPLICATION_LABEL text, + $APP_PACKAGE_NAME text NOT NULL, + $EVENT_TIMESTAMP timestamp with time zone NOT NULL, + $TIMEZONE text, + $USERS text[], + PRIMARY KEY($APP_PACKAGE_NAME, $EVENT_TIMESTAMP) ); """.trimIndent() } From 83cf4fffe0920ecc3a62069a4c5d7f3e8a772c75 Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Sun, 27 Feb 2022 16:27:16 -0800 Subject: [PATCH 49/99] change data type --- .../MigrateChronicleParticipantStats.kt | 50 +++++++++---------- 1 file changed, 23 insertions(+), 27 deletions(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt index b519ea94..75e199ca 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt @@ -1,5 +1,6 @@ package com.openlattice.mechanic.upgrades +import com.geekbeast.postgres.PostgresDatatype import com.geekbeast.rhizome.configuration.RhizomeConfiguration import com.openlattice.authorization.* import com.openlattice.data.requests.NeighborEntityDetails @@ -16,6 +17,7 @@ import com.zaxxer.hikari.HikariConfig import com.zaxxer.hikari.HikariDataSource import org.apache.olingo.commons.api.edm.FullQualifiedName import org.slf4j.LoggerFactory +import java.time.LocalDate import java.time.OffsetDateTime import java.util.* @@ -91,29 +93,23 @@ class MigrateChronicleParticipantStats( private const val PARTICIPANT_ID = "participant_id" private const val ANDROID_FIRST_DATE = "android_first_date" private const val ANDROID_LAST_DATE = "android_last_date" - private const val ANDROID_DATES_COUNT = "android_dates_count" - private const val IOS_FIRST_DATE = "ios_first_date" - private const val IOS_LAST_DATE = "ios_last_date" - private const val IOS_DATES_COUNT = "ios_dates_count" + private const val ANDROID_UNIQUE_DATES = "android_unique_dates" private const val TUD_FIRST_DATE = "tud_first_date" private const val TUD_LAST_DATE = "tud_last_date" - private const val TUD_DATES_COUNT = "tud_dates_count" + private const val TUD_UNIQUE_DATES = "tud_unique_dates" private val CREATE_STATS_TABLE_SQL = """ - CREATE TABLE IF NOT EXISTS participant_stats( + CREATE TABLE IF NOT EXISTS version2_participant_stats( $ORGANIZATION_IO uuid NOT NULL, $V2_STUDY_EKID uuid NOT NULL, $V2_STUDY_ID uuid NOT NULL, $PARTICIPANT_ID text NOT NULL, $ANDROID_FIRST_DATE timestamp with time zone, $ANDROID_LAST_DATE timestamp with time zone, - $ANDROID_DATES_COUNT integer, - $IOS_FIRST_DATE timestamp with time zone, - $IOS_LAST_DATE timestamp with time zone, - $IOS_DATES_COUNT integer, + $ANDROID_UNIQUE_DATES date[], $TUD_FIRST_DATE timestamp with time zone, $TUD_LAST_DATE timestamp with time zone, - $TUD_DATES_COUNT integer, + $TUD_UNIQUE_DATES date[], PRIMARY KEY($V2_STUDY_EKID, $PARTICIPANT_ID) ) """.trimIndent() @@ -125,10 +121,10 @@ class MigrateChronicleParticipantStats( PARTICIPANT_ID, ANDROID_FIRST_DATE, ANDROID_LAST_DATE, - ANDROID_DATES_COUNT, + ANDROID_UNIQUE_DATES, TUD_FIRST_DATE, TUD_LAST_DATE, - TUD_DATES_COUNT + TUD_UNIQUE_DATES ) private val PARTICIPANT_STATS_COLS = COLS.joinToString { it } @@ -141,10 +137,10 @@ class MigrateChronicleParticipantStats( * 4) participantId * 5) androidFirstDate * 6) androidLastDate - * 7) androidDatesCount, + * 7) androidUniqueDates, * 8) tudFirstDate, * 9) tudLastDate - * 10) tudDatesCount + * 10) tudUniqueDates */ private val INSERT_PARTICIPANT_STATS_SQL = """ INSERT INTO participant_stats ($PARTICIPANT_STATS_COLS) values ($PARTICIPANT_STATS_PARAMS) @@ -254,10 +250,10 @@ class MigrateChronicleParticipantStats( ps.setString(++index, it.participantId) ps.setObject(++index, it.androidFirstDate) ps.setObject(++index, it.androidLastDate) - ps.setInt(++index, it.androidDatesCount) + ps.setArray(++index, connection.createArrayOf(PostgresDatatype.DATE.sql(), it.androidUniqueDates.toTypedArray())) ps.setObject(++index, it.tudFirstDate) ps.setObject(++index, it.tudLastDate) - ps.setObject(++index, it.tudDatesCount) + ps.setArray(++index, connection.createArrayOf(PostgresDatatype.DATE.sql(), it.tudUniqueDates.toTypedArray())) ps.addBatch() } ps.executeBatch().sum() @@ -342,10 +338,10 @@ class MigrateChronicleParticipantStats( participantId = participantById.getValue(id).participantId, androidFirstDate = androidStats.first, androidLastDate = androidStats.second, - androidDatesCount = androidStats.third, + androidUniqueDates = androidStats.third, tudFirstDate = tudStats.first, tudLastDate = tudStats.second, - tudDatesCount = tudStats.third + tudUniqueDates = tudStats.third ) }.values.groupBy { it.studyEntityKeyId } } @@ -353,10 +349,10 @@ class MigrateChronicleParticipantStats( // start, end date, count // in theory each participant should only have a single NeighborEntityDetails in the metadata entity set, // but some might have multiple entities - private fun getParticipantAndroidStats(neighbors: List?): Triple { + private fun getParticipantAndroidStats(neighbors: List?): Triple> { if (neighbors == null || neighbors.isEmpty()) { - return Triple(null, null, 0) + return Triple(null, null, setOf()) } val dateTimeStartValues = getOffsetDateTimesFromNeighborEntities(neighbors, DATE_TIME_START_FQN) @@ -369,20 +365,20 @@ class MigrateChronicleParticipantStats( return Triple( first = if (firstDate.isEmpty) null else firstDate.get(), second = if (lastDate.isEmpty) null else lastDate.get(), - third = datesRecorded.map { it.toLocalDate() }.toSet().size // unique dates + third = datesRecorded.map { it.toLocalDate() }.toSet() // unique dates ) } // start date, end date, count - private fun getParticipantTudStats(neighbors: List?): Triple { - if (neighbors == null) return Triple(null, null, 0) + private fun getParticipantTudStats(neighbors: List?): Triple> { + if (neighbors == null) return Triple(null, null, setOf()) val dateTimeValues = getOffsetDateTimesFromNeighborEntities(neighbors, DATETIME_FQN) return Triple( first = dateTimeValues.stream().min(OffsetDateTime::compareTo).get(), second = dateTimeValues.stream().max(OffsetDateTime::compareTo).get(), - third = dateTimeValues.map { it.toLocalDate() }.toSet().size // unique dates + third = dateTimeValues.map { it.toLocalDate() }.toSet() // unique dates ) } @@ -489,10 +485,10 @@ private data class ParticipantStats( val participantId: String, val androidFirstDate: Any?, val androidLastDate: Any?, - val androidDatesCount: Int = 0, + val androidUniqueDates: Set = setOf(), val tudFirstDate: Any?, val tudLastDate: Any?, - val tudDatesCount: Int = 0 + val tudUniqueDates: Set = setOf() ) private data class Study( From 1f6209a6d15a1b5c73ce2e96b87cb5ad12a78dd0 Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Sun, 27 Feb 2022 17:09:22 -0800 Subject: [PATCH 50/99] rename --- .../mechanic/upgrades/MigrateChronicleParticipantStats.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt index 75e199ca..41c923ac 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt @@ -99,7 +99,7 @@ class MigrateChronicleParticipantStats( private const val TUD_UNIQUE_DATES = "tud_unique_dates" private val CREATE_STATS_TABLE_SQL = """ - CREATE TABLE IF NOT EXISTS version2_participant_stats( + CREATE TABLE IF NOT EXISTS v2_participant_stats( $ORGANIZATION_IO uuid NOT NULL, $V2_STUDY_EKID uuid NOT NULL, $V2_STUDY_ID uuid NOT NULL, From d3dd3200672d4f242c410e0019c6f60a0ae60ee0 Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Sun, 27 Feb 2022 17:30:55 -0800 Subject: [PATCH 51/99] fix insert --- .../mechanic/upgrades/MigrateChronicleParticipantStats.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt index 41c923ac..a878b43e 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt @@ -143,7 +143,7 @@ class MigrateChronicleParticipantStats( * 10) tudUniqueDates */ private val INSERT_PARTICIPANT_STATS_SQL = """ - INSERT INTO participant_stats ($PARTICIPANT_STATS_COLS) values ($PARTICIPANT_STATS_PARAMS) + INSERT INTO v2_participant_stats ($PARTICIPANT_STATS_COLS) values ($PARTICIPANT_STATS_PARAMS) ON CONFLICT DO NOTHING """.trimIndent() } From 7cd62de29eb75f0bed03fd997f94ce98b02f1fc0 Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Mon, 28 Feb 2022 13:16:38 -0800 Subject: [PATCH 52/99] fix table definition --- .../MigrateChronicleParticipantStats.kt | 29 ++++++++++++------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt index a878b43e..7442f2d3 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt @@ -86,10 +86,11 @@ class MigrateChronicleParticipantStats( private val DATETIME_FQN = FullQualifiedName("ol.datetime") private val RECORDED_DATE_FQN = FullQualifiedName("ol.recordeddate") + private const val TABLE_NAME = "participant_stats" + // column names private const val ORGANIZATION_IO = "organization_id" - private const val V2_STUDY_ID = "legacy_study_id" - private const val V2_STUDY_EKID = "legacy_study_ekid" + private const val V2_STUDY_ID = "study_id" private const val PARTICIPANT_ID = "participant_id" private const val ANDROID_FIRST_DATE = "android_first_date" private const val ANDROID_LAST_DATE = "android_last_date" @@ -97,26 +98,30 @@ class MigrateChronicleParticipantStats( private const val TUD_FIRST_DATE = "tud_first_date" private const val TUD_LAST_DATE = "tud_last_date" private const val TUD_UNIQUE_DATES = "tud_unique_dates" + private const val IOS_FIRST_DATE = "ios_first_date" + private const val IOS_LAST_DATE = "ios_last_date" + private const val IOS_UNIQUE_DATES = "ios_unique_dates" private val CREATE_STATS_TABLE_SQL = """ - CREATE TABLE IF NOT EXISTS v2_participant_stats( + CREATE TABLE IF NOT EXISTS $TABLE_NAME( $ORGANIZATION_IO uuid NOT NULL, - $V2_STUDY_EKID uuid NOT NULL, $V2_STUDY_ID uuid NOT NULL, $PARTICIPANT_ID text NOT NULL, $ANDROID_FIRST_DATE timestamp with time zone, $ANDROID_LAST_DATE timestamp with time zone, - $ANDROID_UNIQUE_DATES date[], + $ANDROID_UNIQUE_DATES date[] default '{}', $TUD_FIRST_DATE timestamp with time zone, $TUD_LAST_DATE timestamp with time zone, - $TUD_UNIQUE_DATES date[], - PRIMARY KEY($V2_STUDY_EKID, $PARTICIPANT_ID) + $TUD_UNIQUE_DATES date[] default '{}', + $IOS_FIRST_DATE timestamp with time zone, + $IOS_LAST_DATE timestamp with time zone, + $IOS_UNIQUE_DATES date[] default '{}', + PRIMARY KEY($V2_STUDY_ID, $PARTICIPANT_ID) ) """.trimIndent() private val COLS = setOf( ORGANIZATION_IO, - V2_STUDY_EKID, V2_STUDY_ID, PARTICIPANT_ID, ANDROID_FIRST_DATE, @@ -143,7 +148,7 @@ class MigrateChronicleParticipantStats( * 10) tudUniqueDates */ private val INSERT_PARTICIPANT_STATS_SQL = """ - INSERT INTO v2_participant_stats ($PARTICIPANT_STATS_COLS) values ($PARTICIPANT_STATS_PARAMS) + INSERT INTO $TABLE_NAME ($PARTICIPANT_STATS_COLS) values ($PARTICIPANT_STATS_PARAMS) ON CONFLICT DO NOTHING """.trimIndent() } @@ -216,6 +221,11 @@ class MigrateChronicleParticipantStats( edgeEntitySetId = entitySets.getValue(PARTICIPATED_IN_ES) ).toMutableMap() + if (participants.values.isEmpty()) { + logger.info("no participants found in org. Skipping participant stats fetch") + return@forEach + } + logger.info("Retrieved ${participants.values.flatten().size} participants") logger.info("Participant count by study: ${participants.map { studies.getValue(it.key).title to it.value.size }.toMap()}") @@ -245,7 +255,6 @@ class MigrateChronicleParticipantStats( entities.forEach { var index = 0 ps.setObject(++index, it.organizationId) - ps.setObject(++index, it.studyEntityKeyId) ps.setObject(++index, it.studyId) ps.setString(++index, it.participantId) ps.setObject(++index, it.androidFirstDate) From 6147bc1f8992357158e1cb8bc455dfdd4fadca6b Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Sun, 27 Feb 2022 12:53:31 -0800 Subject: [PATCH 53/99] migrate time use diary submisssions --- .../MigrateTimeUseDiarySubmissions.kt | 510 ++++++++++++++++++ 1 file changed, 510 insertions(+) create mode 100644 src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateTimeUseDiarySubmissions.kt diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateTimeUseDiarySubmissions.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateTimeUseDiarySubmissions.kt new file mode 100644 index 00000000..896f2746 --- /dev/null +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateTimeUseDiarySubmissions.kt @@ -0,0 +1,510 @@ +package com.openlattice.mechanic.upgrades + +import com.geekbeast.mappers.mappers.ObjectMappers +import com.geekbeast.rhizome.configuration.RhizomeConfiguration +import com.openlattice.authorization.Principal +import com.openlattice.authorization.PrincipalType +import com.openlattice.data.requests.NeighborEntityDetails +import com.openlattice.data.storage.postgres.PostgresEntityDataQueryService +import com.openlattice.datastore.services.EntitySetManager +import com.openlattice.edm.EdmConstants +import com.openlattice.graph.PagedNeighborRequest +import com.openlattice.hazelcast.HazelcastMap +import com.openlattice.mechanic.Toolbox +import com.openlattice.organizations.roles.SecurePrincipalsManager +import com.openlattice.search.SearchService +import com.openlattice.search.requests.EntityNeighborsFilter +import com.zaxxer.hikari.HikariConfig +import com.zaxxer.hikari.HikariDataSource +import org.apache.olingo.commons.api.edm.FullQualifiedName +import org.slf4j.LoggerFactory +import java.time.OffsetDateTime +import java.util.* + +/** + * @author alfoncenzioka <alfonce@openlattice.com> + */ +class MigrateTimeUseDiarySubmissions( + val toolbox: Toolbox, + private val rhizomeConfiguration: RhizomeConfiguration, + private val dataQueryService: PostgresEntityDataQueryService, + private val entitySetService: EntitySetManager, + private val searchService: SearchService, + private val principalService: SecurePrincipalsManager +) : Upgrade { + + private val entitySetIds: Map = HazelcastMap.ENTITY_SETS.getMap(toolbox.hazelcast).associate { it.value.name to it.key } + private val appConfigs = HazelcastMap.APP_CONFIGS.getMap(toolbox.hazelcast) + + companion object { + private val logger = LoggerFactory.getLogger(MigrateTimeUseDiarySubmissions::class.java) + private val mapper = ObjectMappers.getJsonMapper() + + private const val SUPER_USER_PRINCIPAL_ID = "auth0|5ae9026c04eb0b243f1d2bb6" + + private val SURVEYS_APP_ID = UUID.fromString("bb44218b-515a-4314-b955-df2c991b2575") + + private const val CHRONICLE_APP_ES_PREFIX = "chronicle_" + private const val SURVEYS_APP_ES_PREFIX = "chronicle_surveys_" + + // collection template names + private const val STUDIES_TEMPLATE = "studies" + private const val PARTICIPATED_IN_TEMPLATE = "participatedin" + private const val PARTICIPANTS_TEMPLATE = "participants" + private const val SUBMISSION_TEMPLATE = "submission" + private const val QUESTION_TEMPLATE = "question" + private const val ANSWER_TEMPLATE = "answer" + private const val TIME_RANGE_TEMPLATE = "timerange" + private const val REGISTERED_FOR_TEMPLATE = "registeredfor" + private const val RESPONDS_WITH_TEMPLATE = "respondswith" + private const val ADDRESSES_TEMPLATE = "addresses" + + // entity sets lookup name + private const val STUDIES_ES = "studies" + private const val PARTICIPATED_IN_ES = "participatedIn" + private const val PARTICIPANTS_ES = "participants" + private const val TIME_RANGE_ES = "timeRange" + private const val SUBMISSION_ES = "submission" + private const val QUESTION_ES = "question" + private const val ANSWER_ES = "answer" + private const val REGISTERED_FOR_ES = "registeredFor" + private const val RESPONDED_WITH_ES = "respondedWith" + private const val ADDRESSES_ES = "addresses" + + private val OL_ID_FQN = EdmConstants.ID_FQN + private val STRING_ID_FQN = FullQualifiedName("general.stringid") + private val PERSON_FQN = FullQualifiedName("nc.SubjectIdentification") + private val TITLE_FQN = FullQualifiedName("ol.title") // question title + private val DATE_TIME_START_FQN = FullQualifiedName("ol.datetimestart") //timerange + private val DATE_TIME_END_FQN = FullQualifiedName("ol.datetimeend") + private val VALUES_FQN = FullQualifiedName("ol.values") + private val FULL_NAME_FQN = FullQualifiedName("general.fullname") + private val DATE_TIME_FQN = FullQualifiedName("ol.datetime") + private val ID_FQN = FullQualifiedName("ol.id") // question code + + // column names + private const val STUDY_ID = "study_id" + private const val PARTICIPANT_ID = "participant_id" + private const val ORGANIZATION_ID = "organization_id" + private const val SUBMISSION = "submission" + private const val SUBMISSION_ID = "submission_id" //not unique for each row + private const val SUBMISSION_DATE = "submission_date" + + private const val TABLE_NAME = "time_use_diary_v2" + + private val COLUMNS = linkedSetOf( + STUDY_ID, + ORGANIZATION_ID, + PARTICIPANT_ID, + SUBMISSION_ID, + SUBMISSION, + SUBMISSION_DATE + ).joinToString { it } + + private val CREATE_TABLE_SQL = """ + CREATE TABLE IF NOT EXISTS $TABLE_NAME( + $STUDY_ID uuid not null, + $ORGANIZATION_ID uuid not null, + $SUBMISSION_ID uuid not null, + $PARTICIPANT_ID uuid not null, + $SUBMISSION jsonb not null, + $SUBMISSION_DATE timestamp with time zone not null, + PRIMARY KEY($SUBMISSION_ID, $SUBMISSION_DATE) + ) + """.trimIndent() + + /** + * PreparedStatement bind order + * 1) studyId, + * 2) orgId + * 3) participantId + * 4) submissionId + * 5) submission + * 6) submissionDate + */ + private val INSERT_INTO_TABLE_SQL = """ + INSERT INTO $TABLE_NAME ($COLUMNS) VALUES (?, ?, ?, ?, ?::jsonb, ?) + """.trimIndent() + } + + init { + getHikariDataSource().connection.createStatement().use { statement -> + statement.execute(CREATE_TABLE_SQL) + } + } + + private fun getHikariDataSource(): HikariDataSource { + val (hikariConfiguration) = rhizomeConfiguration.datasourceConfigurations["chronicle"]!! + val hc = HikariConfig(hikariConfiguration) + return HikariDataSource(hc) + } + + private fun getOrgEntitySetNames(orgId: UUID): Map { + val orgIdToStr = orgId.toString().replace("-", "") + val entitySetNameByTemplateName = mapOf( + STUDIES_ES to "$CHRONICLE_APP_ES_PREFIX${orgIdToStr}_$STUDIES_TEMPLATE", + PARTICIPATED_IN_ES to "$CHRONICLE_APP_ES_PREFIX${orgIdToStr}_$PARTICIPATED_IN_TEMPLATE", + PARTICIPANTS_ES to "$CHRONICLE_APP_ES_PREFIX${orgIdToStr}_${PARTICIPANTS_TEMPLATE}", + ADDRESSES_ES to "$SURVEYS_APP_ES_PREFIX${orgIdToStr}_${ADDRESSES_TEMPLATE}", + RESPONDED_WITH_ES to "$SURVEYS_APP_ES_PREFIX${orgIdToStr}_${RESPONDS_WITH_TEMPLATE}", + REGISTERED_FOR_ES to "$SURVEYS_APP_ES_PREFIX${orgIdToStr}_${REGISTERED_FOR_TEMPLATE}", + ANSWER_ES to "$SURVEYS_APP_ES_PREFIX${orgIdToStr}_${ANSWER_TEMPLATE}", + QUESTION_ES to "$SURVEYS_APP_ES_PREFIX${orgIdToStr}_${QUESTION_TEMPLATE}", + SUBMISSION_ES to "$SURVEYS_APP_ES_PREFIX${orgIdToStr}_${SUBMISSION_TEMPLATE}", + TIME_RANGE_ES to "$SURVEYS_APP_ES_PREFIX${orgIdToStr}_${TIME_RANGE_TEMPLATE}" + ) + + return entitySetNameByTemplateName.filter { entitySetIds.keys.contains(it.value) }.mapValues { entitySetIds.getValue(it.value) } + } + + private fun getChronicleSuperUserPrincipals(): Set { + val securablePrincipal = principalService.getSecurablePrincipal(SUPER_USER_PRINCIPAL_ID) + return principalService.getAllPrincipals(securablePrincipal).map { it.principal }.toSet() + Principal(PrincipalType.USER, SUPER_USER_PRINCIPAL_ID) + } + + private fun processOrganization(orgId: UUID, principals: Set): List { + logger.info("Processing org $orgId") + + // get studies + val entitySets = getOrgEntitySetNames(orgId) + val studies: Map = getOrgStudies(entitySetId = entitySets.getValue(STUDIES_ES)) + if (studies.isEmpty()) { + logger.info("organization $orgId has no studies. Skipping") + return listOf() + } + logger.info("Retrieved ${studies.size} studies") + + // study -> participants + val participants: Map> = getOrgParticipants( + participantEntitySetIds = setOf(entitySets.getValue(PARTICIPANTS_ES)), + studiesEntitySetId = entitySets.getValue(STUDIES_ES), + entityKeyIds = studies.keys, + principals = principals, + edgeEntitySetId = entitySets.getValue(PARTICIPATED_IN_ES) + ).toMutableMap() + + if (participants.values.flatten().isEmpty()) { + logger.info("No participants found in $orgId") + return listOf() + } + + // participant -> neighbor entity set id -> [neighbors] + val participantNeighbors: Map>> = getParticipantNeighbors( + entityKeyIds = participants.values.flatten().map { it.id }.toSet(), + entitySetIds = entitySetIds, + principals = principals + ) + + // unique submission. Each submission entity is an entry in the tud submissions table + val submissionsById = participantNeighbors + .asSequence() + .filter { it.value.keys.contains(entitySets.getValue(SUBMISSION_ES)) } + .map { it.value.values.flatten() } + .flatten().associateBy { it.neighborId.get() } + + val answersById = participantNeighbors.values + .asSequence() + .filter { it.keys.contains(entitySets.getValue(ANSWER_ES)) } + .map { it.values.flatten() } + .flatten().associateBy { it.neighborId.get() } + + // answerId -> neighbor esid -> [neighbors] + val answerNeighbors = getAnswerNeighbors( + entityKeyIds = answersById.keys, + entitySetIds = entitySets, + principals = principals + ) + + // submissionId -> [answer] + val answersBySubmissionId = getAnswersBySubmissionId( + entityKeyIds = submissionsById.keys, + entitySetIds = entitySets, + principals = principals + ) + + val participantBySubmissionId = participantNeighbors + .filter { it.value.keys.contains(entitySets.getValue(SUBMISSION_ES)) } + .mapValues { it.value.values.flatten().associate { neighbor -> neighbor.neighborId.get() to it.key } } + .values.first() + + val participantsById = participants.values.flatten().associateBy { it.id } + val studiesById = studies.values.associateBy { it.studyEntityKeyId } + + + return answersBySubmissionId.map { (submissionId, answerEntities) -> + getSubmissionEntity( + orgId, + submissionId, + answerEntities, + participantsById, + participantBySubmissionId, + studiesById, + answerNeighbors.mapValues { answers -> answers.value.mapValues { neighbors -> neighbors.value.first() } }, + submissionsById.getValue(submissionId).neighborDetails.get(), + entitySetIds + ) + } + } + + private fun getSubmissionEntity( + orgId: UUID, + submissionId: UUID, + answers: List, + participantsById: Map, + participantIdBySubmissionId: Map, + studiesById: Map, + answerNeighbors: Map>, + submissionEntity: Map>, + entitySetIds: Map + ): SubmissionEntity { + val dateSubmitted = getFirstValueOrNull(submissionEntity, DATE_TIME_FQN) + + val participantId = participantIdBySubmissionId.getValue(submissionId) + val participant = participantsById.getValue(participantId) + + val responses = answers.map { answer -> + getResponse( + answerId = answer.neighborId.get(), + answerEntity = answer.neighborDetails.get(), + answerNeighbors = answerNeighbors, + entitySetIds = entitySetIds + ) + } + return SubmissionEntity( + orgId = orgId, + submissionId = submissionId, + date = dateSubmitted?.let { OffsetDateTime.parse(it) }, + studyId = studiesById.getValue(participant.studyEntityKeyId).studyId, + participantId = participant.participantId, + responses = responses.filter { it.code != null && it.question != null }.toSet() + ) + } + + + private fun getResponse( + answerId: UUID, + answerEntity: Map>, + answerNeighbors: Map>, + entitySetIds: Map + ): ResponseEntity { + + val responses = getAllValuesOrNull(answerEntity, VALUES_FQN) + val questionEntity = answerNeighbors.getValue(answerId).getValue(entitySetIds.getValue(QUESTION_ES)) + // time range is optional + val timeRangeEntity = answerNeighbors.getValue(answerId)[entitySetIds.getValue(TIME_RANGE_ES)] + val startDateTime = timeRangeEntity?.let { getFirstValueOrNull(it.neighborDetails.get(), DATE_TIME_START_FQN) } + val endDateTime = timeRangeEntity?.let { getFirstValueOrNull(it.neighborDetails.get(), DATE_TIME_END_FQN) } + + + return ResponseEntity( + code = getFirstValueOrNull(questionEntity.neighborDetails.get(), ID_FQN), + question = getFirstValueOrNull(questionEntity.neighborDetails.get(), TITLE_FQN), + response = responses, + startDateTime = startDateTime?.let { OffsetDateTime.parse(it) }, + endDateTime = endDateTime?.let { OffsetDateTime.parse(it) } + ) + } + + private fun getAnswersBySubmissionId( + entityKeyIds: Set, + entitySetIds: Map, + principals: Set + ): Map> { + val registeredForEntitySetId = entitySetIds.getValue(REGISTERED_FOR_ES) + val answerEntitySetId = entitySetIds.getValue(ANSWER_ES) + + val filter = EntityNeighborsFilter( + entityKeyIds, + Optional.of(setOf(answerEntitySetId)), + Optional.empty(), + Optional.of(setOf(registeredForEntitySetId)) + ) + + return searchService.executeEntityNeighborSearch( + setOf(entitySetIds.getValue(SUBMISSION_ES)), + PagedNeighborRequest(filter), + principals + ).neighbors + } + + private fun getAnswerNeighbors( + entityKeyIds: Set, + entitySetIds: Map, + principals: Set + ): Map>> { + val registeredForEntitySetId = entitySetIds.getValue(REGISTERED_FOR_ES) + val submissionEntitySetId = entitySetIds.getValue(SUBMISSION_ES) + val timeRangeEntitySetId = entitySetIds.getValue(TIME_RANGE_ES) + val questionEntitySetId = entitySetIds.getValue(QUESTION_ES) + val addressesEntitySetId = entitySetIds.getValue(ADDRESSES_ES) + + val filter = EntityNeighborsFilter( + entityKeyIds, + Optional.empty(), + Optional.of(setOf(submissionEntitySetId, timeRangeEntitySetId, questionEntitySetId)), + Optional.of(setOf(registeredForEntitySetId, addressesEntitySetId)) + ) + return searchService.executeEntityNeighborSearch( + setOf(entitySetIds.getValue(ANSWER_ES)), + PagedNeighborRequest(filter), + principals + ).neighbors.mapValues { it.value.groupBy { neighbors -> neighbors.neighborEntitySet.get().id } } + } + + private fun getParticipantNeighbors( + entityKeyIds: Set, + entitySetIds: Map, + principals: Set + ): Map>> { + val submissionEntitySetId = entitySetIds.getValue(SUBMISSION_ES) + val respondsWithEntitySetId = entitySetIds.getValue(RESPONDED_WITH_ES) + val answerEntitySetId = entitySetIds.getValue(ANSWER_ES) + + val filter = EntityNeighborsFilter( + entityKeyIds, + Optional.empty(), + Optional.of(setOf(submissionEntitySetId, answerEntitySetId)), + Optional.of(setOf(respondsWithEntitySetId)) + ) + return searchService.executeEntityNeighborSearch( + setOf(entitySetIds.getValue(PARTICIPANTS_ES)), + PagedNeighborRequest(filter), + principals + ).neighbors.mapValues { it.value.groupBy { neighbors -> neighbors.neighborEntitySet.get().id } } + } + + private fun getOrgStudies(entitySetId: UUID): Map { + return dataQueryService.getEntitiesWithPropertyTypeFqns( + mapOf(entitySetId to Optional.empty()), + entitySetService.getPropertyTypesOfEntitySets(setOf(entitySetId)), + mapOf(), + setOf(), + Optional.empty(), + false + ) + .filter { getFirstUUIDOrNull(it.value, STRING_ID_FQN) != null } + .mapValues { getStudyEntity(it.key, it.value) } + } + + private fun getStudyEntity(studyEntityKeyId: UUID, entity: Map>): Study { + val title = getFirstValueOrNull(entity, FULL_NAME_FQN) + val studyId = getFirstUUIDOrNull(entity, STRING_ID_FQN) + return Study(studyEntityKeyId, studyId!!, title) + } + + // Returns a mapping from studyEntityKeyId to list of participants + private fun getOrgParticipants( + participantEntitySetIds: Set, + edgeEntitySetId: UUID, + studiesEntitySetId: UUID, + entityKeyIds: Set, + principals: Set + ) + : Map> { + val filter = EntityNeighborsFilter(entityKeyIds, Optional.of(participantEntitySetIds), Optional.empty(), Optional.of(setOf(edgeEntitySetId))) + + return searchService + .executeEntityNeighborSearch(setOf(studiesEntitySetId), PagedNeighborRequest(filter), principals) + .neighbors + .mapValues { it.value.map { neighbor -> getParticipantFromNeighborEntity(it.key, neighbor) }.toSet() } + + } + + private fun getParticipantFromNeighborEntity(studyEntityKeyId: UUID, entity: NeighborEntityDetails): Participant { + val id = getFirstUUIDOrNull(entity.neighborDetails.get(), OL_ID_FQN) + val participantId = getFirstValueOrNull(entity.neighborDetails.get(), PERSON_FQN) + + return Participant(studyEntityKeyId, id!!, participantId!!) // hope this force unwrapping doesn't throw NPE + + } + + private fun getFirstUUIDOrNull(entity: Map>, fqn: FullQualifiedName): UUID? { + return when (val string = getFirstValueOrNull(entity, fqn)) { + null -> null + else -> UUID.fromString(string) + } + } + + private fun getAllValuesOrNull(entity: Map>, fqn: FullQualifiedName): Set { + entity[fqn]?.let { it -> + return it.mapNotNull { it.toString() }.toSet() + } + return setOf() + } + + private fun getFirstValueOrNull(entity: Map>, fqn: FullQualifiedName): String? { + entity[fqn]?.iterator()?.let { + if (it.hasNext()) return it.next().toString() + } + return null + } + + private fun writeEntitiesToTable(entities: List): Int { + val hds = getHikariDataSource() + + return hds.connection.use { connection -> + try { + val wc = connection.prepareStatement(INSERT_INTO_TABLE_SQL).use { ps -> + entities.forEach { + var index = 0 + ps.setObject(++index, it.studyId) + ps.setObject(++index, it.orgId) + ps.setString(++index, it.participantId) + ps.setObject(++index, it.submissionId) + ps.setString(++index, mapper.writeValueAsString(it.responses)) + ps.setObject(++index, it.date) + ps.addBatch() + } + ps.executeBatch().sum() + } + return@use wc + } catch (ex: Exception) { + return 0 + } + + } + } + + override fun upgrade(): Boolean { + val superUserPrincipals = getChronicleSuperUserPrincipals() + + val orgIds = appConfigs.keys.filter { it.appId == SURVEYS_APP_ID }.map { it.organizationId }.toSet() + val entities = orgIds.map { processOrganization(it, superUserPrincipals) }.flatten() + val written = writeEntitiesToTable(entities) + logger.info("Exported $written entities to $TABLE_NAME") + return true + } + + override fun getSupportedVersion(): Long { + return Version.V2021_07_23.value + } +} + +private data class Study( + val studyEntityKeyId: UUID, + val studyId: UUID, + val title: String? +) + +data class Participant( + val studyEntityKeyId: UUID, + val id: UUID, + val participantId: String, +) + +data class ResponseEntity( + val code: String?, + val question: String?, + val response: Set, + val startDateTime: OffsetDateTime?, + val endDateTime: OffsetDateTime? +) + +data class SubmissionEntity( + val orgId: UUID, + val studyId: UUID, + val submissionId: UUID, + val date: OffsetDateTime?, + val participantId: String, + val responses: Set +) \ No newline at end of file From 95865dfeebb91f91fffaddb0017a638b9d079d97 Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Tue, 1 Mar 2022 23:47:33 -0800 Subject: [PATCH 54/99] add bean to pod --- .../openlattice/mechanic/pods/MechanicUpgradePod.kt | 13 +++++++++++++ .../upgrades/MigrateTimeUseDiarySubmissions.kt | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/pods/MechanicUpgradePod.kt b/src/main/kotlin/com/openlattice/mechanic/pods/MechanicUpgradePod.kt index 3767ea48..7e244aef 100644 --- a/src/main/kotlin/com/openlattice/mechanic/pods/MechanicUpgradePod.kt +++ b/src/main/kotlin/com/openlattice/mechanic/pods/MechanicUpgradePod.kt @@ -72,6 +72,7 @@ import com.openlattice.linking.graph.PostgresLinkingQueryService import com.openlattice.mechanic.MechanicCli.Companion.UPGRADE import com.openlattice.mechanic.Toolbox import com.openlattice.mechanic.upgrades.DeleteOrgMetadataEntitySets +import com.openlattice.mechanic.upgrades.MigrateTimeUseDiarySubmissions import com.openlattice.mechanic.upgrades.V3StudyMigrationUpgrade import com.openlattice.organizations.roles.HazelcastPrincipalService import com.openlattice.organizations.roles.SecurePrincipalsManager @@ -379,6 +380,18 @@ class MechanicUpgradePod { ) } + @Bean + fun migrateTudSubmissions(): MigrateTimeUseDiarySubmissions{ + return MigrateTimeUseDiarySubmissions( + toolbox, + rhizomeConfiguration, + dataQueryService(), + entitySetService(), + searchService(), + principalService() + ) + } + @PostConstruct fun post() { Principals.init(principalService(), hazelcastInstance) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateTimeUseDiarySubmissions.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateTimeUseDiarySubmissions.kt index 896f2746..9d0ca8ff 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateTimeUseDiarySubmissions.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateTimeUseDiarySubmissions.kt @@ -32,7 +32,7 @@ class MigrateTimeUseDiarySubmissions( private val searchService: SearchService, private val principalService: SecurePrincipalsManager ) : Upgrade { - + private val entitySetIds: Map = HazelcastMap.ENTITY_SETS.getMap(toolbox.hazelcast).associate { it.value.name to it.key } private val appConfigs = HazelcastMap.APP_CONFIGS.getMap(toolbox.hazelcast) From d4e58289e8fbc18e82c5bed6faba0d6396ac6dc6 Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Wed, 2 Mar 2022 08:31:47 -0800 Subject: [PATCH 55/99] fix --- .../MigrateTimeUseDiarySubmissions.kt | 163 +++++++++--------- 1 file changed, 80 insertions(+), 83 deletions(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateTimeUseDiarySubmissions.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateTimeUseDiarySubmissions.kt index 9d0ca8ff..5ef374cb 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateTimeUseDiarySubmissions.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateTimeUseDiarySubmissions.kt @@ -30,12 +30,14 @@ class MigrateTimeUseDiarySubmissions( private val dataQueryService: PostgresEntityDataQueryService, private val entitySetService: EntitySetManager, private val searchService: SearchService, - private val principalService: SecurePrincipalsManager + private val principalService: SecurePrincipalsManager, ) : Upgrade { private val entitySetIds: Map = HazelcastMap.ENTITY_SETS.getMap(toolbox.hazelcast).associate { it.value.name to it.key } private val appConfigs = HazelcastMap.APP_CONFIGS.getMap(toolbox.hazelcast) + var totalSubmissionEntities: Int = 0 //keep track of number of submission entities + companion object { private val logger = LoggerFactory.getLogger(MigrateTimeUseDiarySubmissions::class.java) private val mapper = ObjectMappers.getJsonMapper() @@ -106,7 +108,7 @@ class MigrateTimeUseDiarySubmissions( $STUDY_ID uuid not null, $ORGANIZATION_ID uuid not null, $SUBMISSION_ID uuid not null, - $PARTICIPANT_ID uuid not null, + $PARTICIPANT_ID text not null, $SUBMISSION jsonb not null, $SUBMISSION_DATE timestamp with time zone not null, PRIMARY KEY($SUBMISSION_ID, $SUBMISSION_DATE) @@ -165,83 +167,80 @@ class MigrateTimeUseDiarySubmissions( private fun processOrganization(orgId: UUID, principals: Set): List { logger.info("Processing org $orgId") - // get studies - val entitySets = getOrgEntitySetNames(orgId) - val studies: Map = getOrgStudies(entitySetId = entitySets.getValue(STUDIES_ES)) - if (studies.isEmpty()) { - logger.info("organization $orgId has no studies. Skipping") - return listOf() - } - logger.info("Retrieved ${studies.size} studies") - - // study -> participants - val participants: Map> = getOrgParticipants( - participantEntitySetIds = setOf(entitySets.getValue(PARTICIPANTS_ES)), - studiesEntitySetId = entitySets.getValue(STUDIES_ES), - entityKeyIds = studies.keys, - principals = principals, - edgeEntitySetId = entitySets.getValue(PARTICIPATED_IN_ES) - ).toMutableMap() - - if (participants.values.flatten().isEmpty()) { - logger.info("No participants found in $orgId") + + val orgEntitySetIds = getOrgEntitySetNames(orgId) + + // get all participants in studies + val participants: Set = getOrgParticipants( + entitySetIds = orgEntitySetIds, + ) + + if (participants.isEmpty()) { + logger.info("No participants found in org $orgId") return listOf() } // participant -> neighbor entity set id -> [neighbors] val participantNeighbors: Map>> = getParticipantNeighbors( - entityKeyIds = participants.values.flatten().map { it.id }.toSet(), - entitySetIds = entitySetIds, + entityKeyIds = participants.map { it.id }.toSet(), + entitySetIds = orgEntitySetIds, principals = principals ) + val studiesByParticipantId: Map = participantNeighbors + .mapValues { it.value.getOrDefault(orgEntitySetIds.getValue(STUDIES_ES), listOf()).first() } + .mapValues { getStudyEntity(it.value.neighborId.get(), it.value.neighborDetails.get()) } + logger.info("Org studies: ${studiesByParticipantId.values.toSet()}") + // unique submission. Each submission entity is an entry in the tud submissions table val submissionsById = participantNeighbors - .asSequence() - .filter { it.value.keys.contains(entitySets.getValue(SUBMISSION_ES)) } - .map { it.value.values.flatten() } - .flatten().associateBy { it.neighborId.get() } + .values.map { it.getOrDefault(orgEntitySetIds.getValue(SUBMISSION_ES), listOf()) }.flatten().associateBy { it.neighborId.get() } + totalSubmissionEntities += submissionsById.keys.size - val answersById = participantNeighbors.values - .asSequence() - .filter { it.keys.contains(entitySets.getValue(ANSWER_ES)) } - .map { it.values.flatten() } - .flatten().associateBy { it.neighborId.get() } + if (submissionsById.isEmpty()) { + logger.info("no submissions found") + return listOf() + } + + val answersById = participantNeighbors + .values.map { it.getOrDefault(orgEntitySetIds.getValue(ANSWER_ES), listOf()) }.flatten().associateBy { it.neighborId.get() } + if (answersById.isEmpty()) { + logger.warn("unexpected. submission should have answer entities") + return listOf() + } // answerId -> neighbor esid -> [neighbors] val answerNeighbors = getAnswerNeighbors( entityKeyIds = answersById.keys, - entitySetIds = entitySets, + entitySetIds = orgEntitySetIds, principals = principals ) // submissionId -> [answer] val answersBySubmissionId = getAnswersBySubmissionId( entityKeyIds = submissionsById.keys, - entitySetIds = entitySets, + entitySetIds = orgEntitySetIds, principals = principals ) val participantBySubmissionId = participantNeighbors - .filter { it.value.keys.contains(entitySets.getValue(SUBMISSION_ES)) } - .mapValues { it.value.values.flatten().associate { neighbor -> neighbor.neighborId.get() to it.key } } - .values.first() - - val participantsById = participants.values.flatten().associateBy { it.id } - val studiesById = studies.values.associateBy { it.studyEntityKeyId } - + .map { it.value.getOrDefault(orgEntitySetIds.getValue(SUBMISSION_ES), setOf()).associate { neighbor -> neighbor.neighborId.get() to it.key } } + .asSequence() + .flatMap { it.asSequence() } + .groupBy({ it.key }, { it.value }) + .mapValues { it.value.first() } return answersBySubmissionId.map { (submissionId, answerEntities) -> getSubmissionEntity( - orgId, - submissionId, - answerEntities, - participantsById, - participantBySubmissionId, - studiesById, - answerNeighbors.mapValues { answers -> answers.value.mapValues { neighbors -> neighbors.value.first() } }, - submissionsById.getValue(submissionId).neighborDetails.get(), - entitySetIds + orgId = orgId, + submissionId = submissionId, + answerEntities = answerEntities, + participantsById = participants.associateBy { it.id }, + studiesByParticipantId = studiesByParticipantId, + participantBySubmissionId = participantBySubmissionId, + answerNeighbors = answerNeighbors.mapValues { answers -> answers.value.mapValues { neighbors -> neighbors.value.first() } }, + submissionEntity = submissionsById.getValue(submissionId).neighborDetails.get(), + entitySetIds = orgEntitySetIds ) } } @@ -249,20 +248,20 @@ class MigrateTimeUseDiarySubmissions( private fun getSubmissionEntity( orgId: UUID, submissionId: UUID, - answers: List, + answerEntities: List, participantsById: Map, - participantIdBySubmissionId: Map, - studiesById: Map, + studiesByParticipantId: Map, + participantBySubmissionId: Map, answerNeighbors: Map>, submissionEntity: Map>, entitySetIds: Map ): SubmissionEntity { val dateSubmitted = getFirstValueOrNull(submissionEntity, DATE_TIME_FQN) - val participantId = participantIdBySubmissionId.getValue(submissionId) + val participantId = participantBySubmissionId.getValue(submissionId) val participant = participantsById.getValue(participantId) - val responses = answers.map { answer -> + val responses = answerEntities.map { answer -> getResponse( answerId = answer.neighborId.get(), answerEntity = answer.neighborDetails.get(), @@ -274,13 +273,12 @@ class MigrateTimeUseDiarySubmissions( orgId = orgId, submissionId = submissionId, date = dateSubmitted?.let { OffsetDateTime.parse(it) }, - studyId = studiesById.getValue(participant.studyEntityKeyId).studyId, - participantId = participant.participantId, + studyId = studiesByParticipantId.getValue(participantId).studyId, + participantId = participant.participantId!!, //force unwrapping is safe because we have already filtered out "bad" participant entities responses = responses.filter { it.code != null && it.question != null }.toSet() ) } - private fun getResponse( answerId: UUID, answerEntity: Map>, @@ -359,12 +357,14 @@ class MigrateTimeUseDiarySubmissions( val submissionEntitySetId = entitySetIds.getValue(SUBMISSION_ES) val respondsWithEntitySetId = entitySetIds.getValue(RESPONDED_WITH_ES) val answerEntitySetId = entitySetIds.getValue(ANSWER_ES) + val participatedInEntitySetId = entitySetIds.getValue(PARTICIPATED_IN_ES) + val studiesEntitySetId = entitySetIds.getValue(STUDIES_ES) val filter = EntityNeighborsFilter( entityKeyIds, Optional.empty(), - Optional.of(setOf(submissionEntitySetId, answerEntitySetId)), - Optional.of(setOf(respondsWithEntitySetId)) + Optional.of(setOf(submissionEntitySetId, answerEntitySetId, studiesEntitySetId)), + Optional.of(setOf(respondsWithEntitySetId, participatedInEntitySetId)) ) return searchService.executeEntityNeighborSearch( setOf(entitySetIds.getValue(PARTICIPANTS_ES)), @@ -394,28 +394,25 @@ class MigrateTimeUseDiarySubmissions( // Returns a mapping from studyEntityKeyId to list of participants private fun getOrgParticipants( - participantEntitySetIds: Set, - edgeEntitySetId: UUID, - studiesEntitySetId: UUID, - entityKeyIds: Set, - principals: Set - ) - : Map> { - val filter = EntityNeighborsFilter(entityKeyIds, Optional.of(participantEntitySetIds), Optional.empty(), Optional.of(setOf(edgeEntitySetId))) - - return searchService - .executeEntityNeighborSearch(setOf(studiesEntitySetId), PagedNeighborRequest(filter), principals) - .neighbors - .mapValues { it.value.map { neighbor -> getParticipantFromNeighborEntity(it.key, neighbor) }.toSet() } + entitySetIds: Map, + ): Set { + return dataQueryService.getEntitiesWithPropertyTypeFqns( + mapOf(entitySetIds.getValue(PARTICIPANTS_ES) to Optional.empty()), + entitySetService.getPropertyTypesOfEntitySets(setOf(entitySetIds.getValue(PARTICIPANTS_ES))), + mapOf(), + setOf(), + Optional.empty(), + false + ).mapValues { getParticipantEntity(it.key, it.value) }.values.filter { it.participantId != null }.toSet() } - private fun getParticipantFromNeighborEntity(studyEntityKeyId: UUID, entity: NeighborEntityDetails): Participant { - val id = getFirstUUIDOrNull(entity.neighborDetails.get(), OL_ID_FQN) - val participantId = getFirstValueOrNull(entity.neighborDetails.get(), PERSON_FQN) - - return Participant(studyEntityKeyId, id!!, participantId!!) // hope this force unwrapping doesn't throw NPE - + private fun getParticipantEntity(entityKeyId: UUID, entity: Map>): Participant { + val participantId = getFirstValueOrNull(entity, PERSON_FQN) + return Participant( + id = entityKeyId, + participantId = participantId + ) } private fun getFirstUUIDOrNull(entity: Map>, fqn: FullQualifiedName): UUID? { @@ -459,7 +456,7 @@ class MigrateTimeUseDiarySubmissions( } return@use wc } catch (ex: Exception) { - return 0 + throw ex } } @@ -472,6 +469,7 @@ class MigrateTimeUseDiarySubmissions( val entities = orgIds.map { processOrganization(it, superUserPrincipals) }.flatten() val written = writeEntitiesToTable(entities) logger.info("Exported $written entities to $TABLE_NAME") + logger.info("Actual number of entities found in all submission entity sets: $totalSubmissionEntities") return true } @@ -487,9 +485,8 @@ private data class Study( ) data class Participant( - val studyEntityKeyId: UUID, val id: UUID, - val participantId: String, + val participantId: String?, ) data class ResponseEntity( From 6f3ec4bafc488ad03ffd61321ba120c4a252e672 Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Thu, 3 Mar 2022 18:53:31 -0800 Subject: [PATCH 56/99] add time use diary component --- .../mechanic/upgrades/MigrateOrgSettingsToStudies.kt | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateOrgSettingsToStudies.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateOrgSettingsToStudies.kt index 70cbd117..512e4eb7 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateOrgSettingsToStudies.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateOrgSettingsToStudies.kt @@ -159,7 +159,12 @@ class MigrateOrgSettingsToStudies( val studyEntityKeyId = getFirstUUIDOrNull(entity, OL_ID_FQN) val settings: MutableMap = mutableMapOf() - settings[COMPONENTS_SETTING] = appIds.map { appIdToComponentMapping.getValue(it) }.toSet() + + val appComponents = appIds.map { appIdToComponentMapping.getValue(it) }.toMutableSet() + if (appIds.contains(SURVEY_APP_ID)) { + appComponents.add(AppComponents.TIME_USE_DIARY) + } + settings[COMPONENTS_SETTING] = appComponents settings[APP_FREQUENCY_SETTING] = if (orgId == RICE_UNIVERSITY_ORG_ID) AppUsageFrequency.HOURLY else AppUsageFrequency.DAILY return Study( @@ -206,7 +211,8 @@ private data class Study( private enum class AppComponents { CHRONICLE, CHRONICLE_DATA_COLLECTION, - CHRONICLE_SURVEYS + CHRONICLE_SURVEYS, + TIME_USE_DIARY } private enum class AppUsageFrequency { From de04d28267f28cbfb1ce1c49274239d3dd30dc7e Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Thu, 3 Mar 2022 19:31:15 -0800 Subject: [PATCH 57/99] duzz --- .../mechanic/upgrades/MigrateOrgSettingsToStudies.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateOrgSettingsToStudies.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateOrgSettingsToStudies.kt index 512e4eb7..eb5e97e5 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateOrgSettingsToStudies.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateOrgSettingsToStudies.kt @@ -161,7 +161,7 @@ class MigrateOrgSettingsToStudies( val settings: MutableMap = mutableMapOf() val appComponents = appIds.map { appIdToComponentMapping.getValue(it) }.toMutableSet() - if (appIds.contains(SURVEY_APP_ID)) { + if (appIds.contains(SURVEY_APP_ID) && orgId != LEGACY_ORG_ID) { appComponents.add(AppComponents.TIME_USE_DIARY) } settings[COMPONENTS_SETTING] = appComponents From 1aa8dc4703cce0a398d86a3bcd40632e2873cd62 Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Fri, 4 Mar 2022 10:02:57 -0800 Subject: [PATCH 58/99] export summarized time use diary data --- .../MigrateTimeUseDiarySummarizedData.kt | 203 ++++++++++++++++++ 1 file changed, 203 insertions(+) create mode 100644 src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateTimeUseDiarySummarizedData.kt diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateTimeUseDiarySummarizedData.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateTimeUseDiarySummarizedData.kt new file mode 100644 index 00000000..b10d2458 --- /dev/null +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateTimeUseDiarySummarizedData.kt @@ -0,0 +1,203 @@ +package com.openlattice.mechanic.upgrades + +import com.fasterxml.jackson.databind.json.JsonMapper +import com.geekbeast.mappers.mappers.ObjectMappers +import com.geekbeast.rhizome.configuration.RhizomeConfiguration +import com.geekbeast.util.log +import com.openlattice.authorization.AuthorizationManager +import com.openlattice.authorization.Principal +import com.openlattice.authorization.PrincipalType +import com.openlattice.data.requests.NeighborEntityDetails +import com.openlattice.data.storage.postgres.PostgresEntityDataQueryService +import com.openlattice.datastore.services.EntitySetManager +import com.openlattice.graph.PagedNeighborRequest +import com.openlattice.hazelcast.HazelcastMap +import com.openlattice.mechanic.Toolbox +import com.openlattice.organizations.roles.SecurePrincipalsManager +import com.openlattice.search.SearchService +import com.openlattice.search.requests.EntityNeighborsFilter +import com.zaxxer.hikari.HikariConfig +import com.zaxxer.hikari.HikariDataSource +import org.apache.olingo.commons.api.edm.FullQualifiedName +import org.slf4j.LoggerFactory +import java.time.OffsetDateTime +import java.util.* + +/** + * @author alfoncenzioka <alfonce@openlattice.com> + */ +class MigrateTimeUseDiarySummarizedData( + toolbox: Toolbox, + private val rhizomeConfiguration: RhizomeConfiguration, + private val principalService: SecurePrincipalsManager, + private val searchService: SearchService, + private val dataQueryService: PostgresEntityDataQueryService, + private val entitySetService: EntitySetManager +) : Upgrade{ + private val entitySetIds: Map = HazelcastMap.ENTITY_SETS.getMap(toolbox.hazelcast).associate { it.value.name to it.key } + private val appConfigs = HazelcastMap.APP_CONFIGS.getMap(toolbox.hazelcast) + + companion object { + private val logger = LoggerFactory.getLogger(MigrateTimeUseDiarySummarizedData::class.java) + private val mapper = ObjectMappers.getJsonMapper() + + private const val SUPER_USER_PRINCIPAL_ID = "auth0|5ae9026c04eb0b243f1d2bb6" + private val SURVEYS_APP_ID = UUID.fromString("bb44218b-515a-4314-b955-df2c991b2575") + + private const val SURVEYS_APP_ES_PREFIX = "chronicle_surveys_" + + private const val SUBMISSION_TEMPLATE = "submission" + private const val SUMMARY_TEMPLATE = "summaryset" + private const val REGISTERED_FOR_TEMPLATE = "registeredfor" + + private const val SUBMISSION_ES = "submission" + private const val SUMMARY_ES = "summary" + private const val REGISTERED_FOR_ES = "registeredFor" + + private val VARIABLE_FQN = FullQualifiedName("ol.variable") + private val VALUES_FQN = FullQualifiedName("ol.values") + private val DATE_COMPLETED = FullQualifiedName("date.completeddatetime") + + private val CREATE_TABLE_SQL = """ + CREATE TABLE IF NOT EXISTS time_use_diary_summary( + submission_id uuid not null, + date timestamp with time zone not null, + data jsonb not null, + PRIMARY KEY (submission_id) + ) + """.trimIndent() + + private val INSERT_INTO_TABLE_SQL = """ + INSERT INTO time_use_diary_summary values(?, ?, ?::jsonb) + """.trimIndent() + } + + + init { + getHikariDataSource().connection.createStatement().use { statement -> + statement.execute(CREATE_TABLE_SQL) + } + } + + override fun upgrade(): Boolean { + val orgIds = appConfigs.keys.filter { it.appId == SURVEYS_APP_ID }.map { it.organizationId }.toSet() + val principals = getChronicleSuperUserPrincipals() + val entities = orgIds.map { getEntitiesForOrg(it, principals) }.flatten().toSet() + + val written = writeEntitiesToTable(entities) + logger.info("Wrote $written entities to table") + return true + } + + override fun getSupportedVersion(): Long { + return Version.V2021_07_23.value + } + + private fun writeEntitiesToTable(entities: Set): Int { + val hds = getHikariDataSource() + return hds.connection.use { connection -> + val wc = connection.prepareStatement(INSERT_INTO_TABLE_SQL).use { ps -> + entities.forEach { + ps.setObject(1, it.submissionId) + ps.setObject(2, it.date) + ps.setString(3, mapper.writeValueAsString(it.entities)) + ps.addBatch() + } + ps.executeBatch().sum() + } + return@use wc + } + } + + + private fun getEntitiesForOrg(orgId: UUID, principals: Set): Set { + val entitySets = getOrgEntitySetNames(orgId) + val submissionEntitySetId = entitySetIds.getValue(SUBMISSION_ES) + val registeredForEntitySetId = entitySetIds.getValue(REGISTERED_FOR_ES) + val summaryEntitySetId = entitySets.getValue(SUMMARY_ES) + + val submissionIds = dataQueryService.getEntitiesWithPropertyTypeFqns( + mapOf(submissionEntitySetId to Optional.empty()), + entitySetService.getPropertyTypesOfEntitySets(setOf(submissionEntitySetId)), + mapOf(), + setOf(), + Optional.empty(), + false + ).keys + + // get entities from summarized entity set associated with submission ids + val filter = EntityNeighborsFilter( + submissionIds, + Optional.of(setOf(summaryEntitySetId)), + Optional.empty(), + Optional.of(setOf(registeredForEntitySetId)) + ) + val searchResult = searchService.executeEntityNeighborSearch( + setOf(submissionEntitySetId), + PagedNeighborRequest(filter), + principals + ).neighbors.mapValues { getSummaryEntityForSubmission(it.key, it.value) } + + return searchResult.values.toSet() + } + + private fun getFirstValueOrNull(entity: Map>, fqn: FullQualifiedName): String? { + entity[fqn]?.iterator()?.let { + if (it.hasNext()) return it.next().toString() + } + return null + } + + private fun getSummaryEntityForSubmission(submissionId: UUID, neighbors: List): SubmissionEntity { + val date = getFirstValueOrNull(neighbors.first().associationDetails, DATE_COMPLETED) + + val values = neighbors.map { + val entity = it.neighborDetails.get() + SummarizedEntity( + variable = getFirstValueOrNull(entity, VARIABLE_FQN), + value = getFirstValueOrNull(entity, VALUES_FQN) + ) + }.filter { it.value != null && it.variable != null }.toSet() + + return SubmissionEntity( + submissionId = submissionId, + entities = values, + date = OffsetDateTime.parse(date) + ) + } + + private fun getOrgEntitySetNames(orgId: UUID): Map { + val orgIdToStr = orgId.toString().replace("-", "") + val entitySetNameByTemplateName = mapOf( + REGISTERED_FOR_ES to "$SURVEYS_APP_ES_PREFIX${orgIdToStr}_${REGISTERED_FOR_TEMPLATE}", + SUBMISSION_ES to "$SURVEYS_APP_ES_PREFIX${orgIdToStr}_${SUBMISSION_TEMPLATE}", + SUMMARY_ES to "$SURVEYS_APP_ES_PREFIX${orgIdToStr}_${SUMMARY_TEMPLATE}" + ) + + return entitySetNameByTemplateName.filter { entitySetIds.keys.contains(it.value) }.mapValues { entitySetIds.getValue(it.value) } + } + + + private fun getHikariDataSource(): HikariDataSource { + val (hikariConfiguration) = rhizomeConfiguration.datasourceConfigurations["chronicle"]!! + val hc = HikariConfig(hikariConfiguration) + return HikariDataSource(hc) + } + + private fun getChronicleSuperUserPrincipals(): Set { + val securablePrincipal = principalService.getSecurablePrincipal(SUPER_USER_PRINCIPAL_ID) + return principalService.getAllPrincipals(securablePrincipal).map { it.principal }.toSet() + Principal(PrincipalType.USER, SUPER_USER_PRINCIPAL_ID) + } + +} + +private data class SummarizedEntity( + val variable: String?, + val value: String? +) + +private data class SubmissionEntity( + val submissionId: UUID, + val entities: Set, + val date: OffsetDateTime +) From 010962aee14a52c643027ecea9c0ab6f72cd7e9f Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Fri, 4 Mar 2022 10:05:33 -0800 Subject: [PATCH 59/99] add to pod --- .../openlattice/mechanic/pods/MechanicUpgradePod.kt | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/main/kotlin/com/openlattice/mechanic/pods/MechanicUpgradePod.kt b/src/main/kotlin/com/openlattice/mechanic/pods/MechanicUpgradePod.kt index 9d03d914..8e3e4326 100644 --- a/src/main/kotlin/com/openlattice/mechanic/pods/MechanicUpgradePod.kt +++ b/src/main/kotlin/com/openlattice/mechanic/pods/MechanicUpgradePod.kt @@ -73,6 +73,7 @@ import com.openlattice.mechanic.MechanicCli.Companion.UPGRADE import com.openlattice.mechanic.Toolbox import com.openlattice.mechanic.upgrades.DeleteOrgMetadataEntitySets import com.openlattice.mechanic.upgrades.ExportOrganizationMembers +import com.openlattice.mechanic.upgrades.MigrateTimeUseDiarySummarizedData import com.openlattice.mechanic.upgrades.V3StudyMigrationUpgrade import com.openlattice.organizations.roles.HazelcastPrincipalService import com.openlattice.organizations.roles.SecurePrincipalsManager @@ -385,6 +386,18 @@ class MechanicUpgradePod { return ExportOrganizationMembers(toolbox, hikariDataSource, principalService(), hazelcastInstance) } + @Bean + fun migrateTImeUseDiarySummarizedData() : MigrateTimeUseDiarySummarizedData { + return MigrateTimeUseDiarySummarizedData( + toolbox, + rhizomeConfiguration, + principalService(), + searchService(), + dataQueryService(), + entitySetService() + ) + } + @PostConstruct fun post() { Principals.init(principalService(), hazelcastInstance) From ab14af525814d1214e2018083c1eee57f22753d0 Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Fri, 4 Mar 2022 10:23:01 -0800 Subject: [PATCH 60/99] add logging --- .../mechanic/upgrades/MigrateTimeUseDiarySummarizedData.kt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateTimeUseDiarySummarizedData.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateTimeUseDiarySummarizedData.kt index b10d2458..a76ba204 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateTimeUseDiarySummarizedData.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateTimeUseDiarySummarizedData.kt @@ -111,7 +111,10 @@ class MigrateTimeUseDiarySummarizedData( private fun getEntitiesForOrg(orgId: UUID, principals: Set): Set { + logger.info("processing org $orgId") val entitySets = getOrgEntitySetNames(orgId) + logger.info("entity sets: $entitySets") + val submissionEntitySetId = entitySetIds.getValue(SUBMISSION_ES) val registeredForEntitySetId = entitySetIds.getValue(REGISTERED_FOR_ES) val summaryEntitySetId = entitySets.getValue(SUMMARY_ES) From 7d79cc51552e893d3a1b8a461492174c3ebdd468 Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Fri, 4 Mar 2022 11:45:30 -0800 Subject: [PATCH 61/99] fix --- .../mechanic/upgrades/MigrateTimeUseDiarySummarizedData.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateTimeUseDiarySummarizedData.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateTimeUseDiarySummarizedData.kt index a76ba204..c42282cc 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateTimeUseDiarySummarizedData.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateTimeUseDiarySummarizedData.kt @@ -115,8 +115,8 @@ class MigrateTimeUseDiarySummarizedData( val entitySets = getOrgEntitySetNames(orgId) logger.info("entity sets: $entitySets") - val submissionEntitySetId = entitySetIds.getValue(SUBMISSION_ES) - val registeredForEntitySetId = entitySetIds.getValue(REGISTERED_FOR_ES) + val submissionEntitySetId = entitySets.getValue(SUBMISSION_ES) + val registeredForEntitySetId = entitySets.getValue(REGISTERED_FOR_ES) val summaryEntitySetId = entitySets.getValue(SUMMARY_ES) val submissionIds = dataQueryService.getEntitiesWithPropertyTypeFqns( From e057a625f1a0152e9f2e4102cf79b67da8ef7a4a Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Fri, 4 Mar 2022 12:09:20 -0800 Subject: [PATCH 62/99] log dates --- .../mechanic/upgrades/MigrateTimeUseDiarySummarizedData.kt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateTimeUseDiarySummarizedData.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateTimeUseDiarySummarizedData.kt index c42282cc..c107939a 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateTimeUseDiarySummarizedData.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateTimeUseDiarySummarizedData.kt @@ -153,6 +153,8 @@ class MigrateTimeUseDiarySummarizedData( private fun getSummaryEntityForSubmission(submissionId: UUID, neighbors: List): SubmissionEntity { val date = getFirstValueOrNull(neighbors.first().associationDetails, DATE_COMPLETED) + val dates = neighbors.map { getFirstValueOrNull(it.associationDetails, DATE_COMPLETED) } + logger.info("dates: $dates") val values = neighbors.map { val entity = it.neighborDetails.get() From a48e61f9a0e278e7dc82dec1ad6af79d1f7f396e Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Fri, 4 Mar 2022 12:35:20 -0800 Subject: [PATCH 63/99] missing fqn from association entity --- .../upgrades/MigrateTimeUseDiarySummarizedData.kt | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateTimeUseDiarySummarizedData.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateTimeUseDiarySummarizedData.kt index c107939a..050c624a 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateTimeUseDiarySummarizedData.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateTimeUseDiarySummarizedData.kt @@ -56,19 +56,17 @@ class MigrateTimeUseDiarySummarizedData( private val VARIABLE_FQN = FullQualifiedName("ol.variable") private val VALUES_FQN = FullQualifiedName("ol.values") - private val DATE_COMPLETED = FullQualifiedName("date.completeddatetime") private val CREATE_TABLE_SQL = """ CREATE TABLE IF NOT EXISTS time_use_diary_summary( submission_id uuid not null, - date timestamp with time zone not null, data jsonb not null, PRIMARY KEY (submission_id) ) """.trimIndent() private val INSERT_INTO_TABLE_SQL = """ - INSERT INTO time_use_diary_summary values(?, ?, ?::jsonb) + INSERT INTO time_use_diary_summary values(?, ?::jsonb) """.trimIndent() } @@ -99,7 +97,6 @@ class MigrateTimeUseDiarySummarizedData( val wc = connection.prepareStatement(INSERT_INTO_TABLE_SQL).use { ps -> entities.forEach { ps.setObject(1, it.submissionId) - ps.setObject(2, it.date) ps.setString(3, mapper.writeValueAsString(it.entities)) ps.addBatch() } @@ -152,10 +149,6 @@ class MigrateTimeUseDiarySummarizedData( } private fun getSummaryEntityForSubmission(submissionId: UUID, neighbors: List): SubmissionEntity { - val date = getFirstValueOrNull(neighbors.first().associationDetails, DATE_COMPLETED) - val dates = neighbors.map { getFirstValueOrNull(it.associationDetails, DATE_COMPLETED) } - logger.info("dates: $dates") - val values = neighbors.map { val entity = it.neighborDetails.get() SummarizedEntity( @@ -167,7 +160,6 @@ class MigrateTimeUseDiarySummarizedData( return SubmissionEntity( submissionId = submissionId, entities = values, - date = OffsetDateTime.parse(date) ) } @@ -204,5 +196,4 @@ private data class SummarizedEntity( private data class SubmissionEntity( val submissionId: UUID, val entities: Set, - val date: OffsetDateTime ) From 2e5431f54a266fd346115c031c69268a82270d59 Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Fri, 4 Mar 2022 13:10:10 -0800 Subject: [PATCH 64/99] check for nulls --- .../upgrades/MigrateTimeUseDiarySummarizedData.kt | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateTimeUseDiarySummarizedData.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateTimeUseDiarySummarizedData.kt index 050c624a..1f49c42d 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateTimeUseDiarySummarizedData.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateTimeUseDiarySummarizedData.kt @@ -111,10 +111,16 @@ class MigrateTimeUseDiarySummarizedData( logger.info("processing org $orgId") val entitySets = getOrgEntitySetNames(orgId) logger.info("entity sets: $entitySets") - - val submissionEntitySetId = entitySets.getValue(SUBMISSION_ES) - val registeredForEntitySetId = entitySets.getValue(REGISTERED_FOR_ES) - val summaryEntitySetId = entitySets.getValue(SUMMARY_ES) + + + val submissionEntitySetId = entitySets[SUBMISSION_ES] + val registeredForEntitySetId = entitySets[REGISTERED_FOR_ES] + val summaryEntitySetId = entitySets[SUMMARY_ES] + + if (submissionEntitySetId == null || registeredForEntitySetId == null || summaryEntitySetId == null) { + logger.info("submission: {}, registered_for: {}, summary: {}", submissionEntitySetId, registeredForEntitySetId, summaryEntitySetId) + return setOf() + } val submissionIds = dataQueryService.getEntitiesWithPropertyTypeFqns( mapOf(submissionEntitySetId to Optional.empty()), From b9445bd779d239a221c3261e16211a8d3dde4b46 Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Fri, 4 Mar 2022 13:32:09 -0800 Subject: [PATCH 65/99] fix filtered search --- .../mechanic/upgrades/MigrateTimeUseDiarySummarizedData.kt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateTimeUseDiarySummarizedData.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateTimeUseDiarySummarizedData.kt index 1f49c42d..5d1b9e40 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateTimeUseDiarySummarizedData.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateTimeUseDiarySummarizedData.kt @@ -131,6 +131,11 @@ class MigrateTimeUseDiarySummarizedData( false ).keys + if (submissionIds.isEmpty()) { + logger.info("no submission entities found") + return setOf() + } + // get entities from summarized entity set associated with submission ids val filter = EntityNeighborsFilter( submissionIds, From 4abdb822c1d3f16d798b140086c4bd0c11f6d4fa Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Fri, 4 Mar 2022 14:06:44 -0800 Subject: [PATCH 66/99] fix sql --- .../mechanic/upgrades/MigrateTimeUseDiarySummarizedData.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateTimeUseDiarySummarizedData.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateTimeUseDiarySummarizedData.kt index 5d1b9e40..1db51be5 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateTimeUseDiarySummarizedData.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateTimeUseDiarySummarizedData.kt @@ -97,7 +97,7 @@ class MigrateTimeUseDiarySummarizedData( val wc = connection.prepareStatement(INSERT_INTO_TABLE_SQL).use { ps -> entities.forEach { ps.setObject(1, it.submissionId) - ps.setString(3, mapper.writeValueAsString(it.entities)) + ps.setString(2, mapper.writeValueAsString(it.entities)) ps.addBatch() } ps.executeBatch().sum() From f39d4cff0f42aa44482e3a48957b7a0a5f978fa4 Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Fri, 4 Mar 2022 17:51:18 -0800 Subject: [PATCH 67/99] import preprocessed data --- .../mechanic/pods/MechanicUpgradePod.kt | 13 + .../upgrades/MigratePreprocessedData.kt | 398 ++++++++++++++++++ 2 files changed, 411 insertions(+) create mode 100644 src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt diff --git a/src/main/kotlin/com/openlattice/mechanic/pods/MechanicUpgradePod.kt b/src/main/kotlin/com/openlattice/mechanic/pods/MechanicUpgradePod.kt index 9d03d914..152b226f 100644 --- a/src/main/kotlin/com/openlattice/mechanic/pods/MechanicUpgradePod.kt +++ b/src/main/kotlin/com/openlattice/mechanic/pods/MechanicUpgradePod.kt @@ -73,6 +73,7 @@ import com.openlattice.mechanic.MechanicCli.Companion.UPGRADE import com.openlattice.mechanic.Toolbox import com.openlattice.mechanic.upgrades.DeleteOrgMetadataEntitySets import com.openlattice.mechanic.upgrades.ExportOrganizationMembers +import com.openlattice.mechanic.upgrades.MigratePreprocessedData import com.openlattice.mechanic.upgrades.V3StudyMigrationUpgrade import com.openlattice.organizations.roles.HazelcastPrincipalService import com.openlattice.organizations.roles.SecurePrincipalsManager @@ -385,6 +386,18 @@ class MechanicUpgradePod { return ExportOrganizationMembers(toolbox, hikariDataSource, principalService(), hazelcastInstance) } + @Bean + fun migratePreprocessedData() : MigratePreprocessedData { + return MigratePreprocessedData( + toolbox, + rhizomeConfiguration, + dataQueryService(), + entitySetService(), + searchService(), + principalService() + ) + } + @PostConstruct fun post() { Principals.init(principalService(), hazelcastInstance) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt new file mode 100644 index 00000000..9df7f0ad --- /dev/null +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt @@ -0,0 +1,398 @@ +package com.openlattice.mechanic.upgrades + +import com.geekbeast.postgres.streams.BasePostgresIterable +import com.geekbeast.postgres.streams.PreparedStatementHolderSupplier +import com.geekbeast.rhizome.configuration.RhizomeConfiguration +import com.openlattice.authorization.Principal +import com.openlattice.authorization.PrincipalType +import com.openlattice.data.requests.NeighborEntityDetails +import com.openlattice.data.storage.postgres.PostgresEntityDataQueryService +import com.openlattice.datastore.services.EntitySetManager +import com.openlattice.graph.PagedNeighborRequest +import com.openlattice.hazelcast.HazelcastMap +import com.openlattice.mechanic.Toolbox +import com.openlattice.organizations.roles.SecurePrincipalsManager +import com.openlattice.search.SearchService +import com.openlattice.search.requests.EntityNeighborsFilter +import com.zaxxer.hikari.HikariConfig +import com.zaxxer.hikari.HikariDataSource +import org.apache.olingo.commons.api.edm.FullQualifiedName +import org.slf4j.LoggerFactory +import java.sql.ResultSet +import java.time.OffsetDateTime +import java.util.* + +/** + * @author alfoncenzioka <alfonce@openlattice.com> + */ +class MigratePreprocessedData( + val toolbox: Toolbox, + private val rhizomeConfiguration: RhizomeConfiguration, + private val dataQueryService: PostgresEntityDataQueryService, + private val entitySetService: EntitySetManager, + private val searchService: SearchService, + private val principalService: SecurePrincipalsManager, +) : Upgrade { + + private val entitySetIds: Map = HazelcastMap.ENTITY_SETS.getMap(toolbox.hazelcast).associate { it.value.name to it.key } + private val appConfigs = HazelcastMap.APP_CONFIGS.getMap(toolbox.hazelcast) + + companion object { + private val logger = LoggerFactory.getLogger(MigratePreprocessedData::class.java) + + private const val SUPER_USER_PRINCIPAL_ID = "auth0|5ae9026c04eb0b243f1d2bb6" + + private val DATA_COLLECTION_APP_ID = UUID.fromString("c4e6d8fd-daf9-41e7-8c59-2a12c7ee0857") + private val LEGACY_ORG_ID = UUID.fromString("7349c446-2acc-4d14-b2a9-a13be39cff93") + + private const val CHRONICLE_APP_ES_PREFIX = "chronicle_" + private const val DATA_COLLECTION_APP_ES_PREFIX = "chronicle_data_collection_" + + private const val LEGACY_STUDIES_ES = "chronicle_study" + private const val LEGACY_RECORDED_BY_ES = "chronicle_recorded_by" + private const val LEGACY_PREPROCESSED_ES = "chronicle_preprocessed_app_data" + private const val LEGACY_PARTICIPATED_IN_ES = "chronicle_participated_in" + + + // collection template names + private const val PRE_PROCESSED_TEMPLATE = "preprocesseddata"; + private const val STUDIES_TEMPLATE = "studies" + private const val PARTICIPATED_IN_TEMPLATE = "participatedin" + private const val PARTICIPANTS_TEMPLATE = "participants" + private const val RECORDED_BY_TEMPLATE = "recordedby" + + // entity sets lookup name + private const val RECORDED_BY_ES = "recordedBy" + private const val PRE_PROCESSED_ES = "preprocessed" + private const val STUDIES_ES = "studies" + private const val PARTICIPATED_IN_ES = "participatedIn" + private const val PARTICIPANTS_ES = "participants" + + // table columns + private const val PARTICIPANT_ID = "participant_id" + private const val STUDY_ID = "study_id" + private const val APP_LABEL = "app_label" + private const val DATE_TIME_START = "datetime_start" + private const val DATE_TIME_END = "datetime_end" + private const val APP_PACKAGE_NAME = "app_package_name" + private const val TIMEZONE = "timezone" + private const val RECORD_TYPE ="record_type" + private const val NEW_PERIOD ="new_period" + private const val NEW_APP = "new_app" + private const val DURATION = "duration_seconds" + private const val WARNING ="warning" + + private val PERSON_FQN = FullQualifiedName("nc.SubjectIdentification") + private val TITLE_FQN = FullQualifiedName("ol.title") + private val DATE_TIME_START_FQN = FullQualifiedName("ol.datetimestart") + private val DATE_TIME_END_FQN = FullQualifiedName("general.EndTime") + private val FULL_NAME_FQN = FullQualifiedName("general.fullname") + private val TIMEZONE_FQN = FullQualifiedName("ol.timezone") + private val RECORD_TYPE_FQN = FullQualifiedName("ol.recordtype") + private val NEW_PERIOD_FQN = FullQualifiedName("ol.newperiod") + private val DURATION_FQN = FullQualifiedName("general.Duration") + private val WARNING_FQN = FullQualifiedName("ol.warning") + private val NEW_APP_FQN = FullQualifiedName("ol.newapp") + + + private val CREATE_TABLE_SQL = """ + CREATE TABLE IF NOT EXISTS preprocessed_usage_events( + $STUDY_ID uuid not null, + $PARTICIPANT_ID text not null, + $APP_LABEL text, + $APP_PACKAGE_NAME text not null, + $DATE_TIME_START timestamp with time zone, + $DATE_TIME_END timestamp with time zone, + $TIMEZONE text not null, + $RECORD_TYPE text not null, + $NEW_PERIOD boolean, + $NEW_APP boolean, + $DURATION double, + $WARNING text + PRIMARY_KEY($PARTICIPANT_ID, $STUDY_ID, $DATE_TIME_START, $APP_PACKAGE_NAME) + ) + """.trimIndent() + + private val PARAMS_BINDING = linkedSetOf( + STUDY_ID, + PARTICIPANT_ID, + APP_LABEL, + APP_PACKAGE_NAME, + DATE_TIME_START, + DATE_TIME_END, + TIMEZONE, + RECORD_TYPE, + NEW_PERIOD, + NEW_APP, + DURATION, + WARNING + ).joinToString { "?" } + + /** + * PreparedStatement bind order + * 1) legacy study id + * 2) participant_id + * 3) app label + * 4) appPackageName + * 5) dateStart + * 6) dateEnd, + * 7) timezone + * 8) recordType + * 9) newPeriod + * 10) newApp + * 11) duration + * 12) warning + */ + private val INSERT_SQL = """ + INSERT INTO preprocessed_usage_events values ($PARAMS_BINDING) + """.trimIndent() + } + + init { + getHikariDataSource().connection.createStatement().use { statement -> + statement.execute(CREATE_TABLE_SQL) + } + } + + override fun upgrade(): Boolean { + + val participants = BasePostgresIterable( + PreparedStatementHolderSupplier( + getHikariDataSource(), + "SELECT * FROM participant_export" + ) {} + ) { + participant(it) + }.associateBy { it.participant_ek_id } + + val orgIds = (appConfigs.keys.filter { it.appId == DATA_COLLECTION_APP_ID }.map { it.organizationId } + LEGACY_ORG_ID).toSet() + val principals = getChronicleSuperUserPrincipals() + val entitiesToWrite = orgIds.associateWith { getEntitiesForOrg(it, participants, principals) }.values.flatten() + + val written = writeEntities(entitiesToWrite) + logger.info("Exported $written entities to preprocessed table") + return true + } + + override fun getSupportedVersion(): Long { + return Version.V2021_07_23.value + } + + private fun writeEntities(entities: List): Int { + return getHikariDataSource().connection.use { connection -> + try { + val wc = connection.prepareStatement(INSERT_SQL).use { ps -> + entities.forEach { + var index = 0 + ps.setObject(++index, it.study_id) + ps.setString(++index, it.participant_id) + ps.setString(++index, it.appLabel) + ps.setString(++index, it.packageName) + ps.setObject(++index, it.datetimeStart) + ps.setObject(++index, it.datetimeEnd) + ps.setString(++index, it.timezone) + ps.setString(++index, it.recordType) + ps.setBoolean(++index, it.newPeriod) + ps.setBoolean(++index, it.newApp) + ps.setObject(++index, it.duration) + ps.setString(++index, it.warning) + } + ps.executeBatch().sum() + } + return@use wc + } catch (ex: Exception) { + logger.error("exception", ex) + throw ex + } + } + } + + private fun getHikariDataSource(): HikariDataSource { + val (hikariConfiguration) = rhizomeConfiguration.datasourceConfigurations["chronicle"]!! + val hc = HikariConfig(hikariConfiguration) + return HikariDataSource(hc) + } + + private fun participant(rs: ResultSet): ParticipantExport { + return ParticipantExport( + participant_ek_id = rs.getObject("participant_ek_id", UUID::class.java), + study_id = rs.getObject("study_id", UUID::class.java), + organization_id = rs.getObject("organization_id", UUID::class.java), + study_ek_id = rs.getObject("study_ek_id", UUID::class.java), + study_es_id = rs.getObject("study_es_id", UUID::class.java), + legacy_participant_id = rs.getString("legacy_participant_id"), + participant_es_id = rs.getObject("participant_es_id", UUID::class.java) + ) + } + + private fun getEntitiesForOrg(orgId: UUID, participants: Map, principals: Set): List { + logger.info("getting preprocessed data entities for org $orgId") + val orgEntitySetIds = getOrgEntitySetNames(orgId) + + val participantEntitySetIds = when(orgId) { + LEGACY_ORG_ID -> getLegacyParticipantEntitySetIds() + else -> setOf(orgEntitySetIds.getValue(PARTICIPANTS_ES)) + } + val orgParticipants: Set = getOrgParticipants(participantEntitySetIds) + if (orgParticipants.isEmpty()) { + logger.info("No participants found. Skipping org") + return listOf() + } + + val participantNeighbors: Map> = getParticipantNeighbors( + entityKeyIds = orgParticipants.map { it.id }.toSet(), + entitySetIds = orgEntitySetIds, + participantEntitySetIds = participantEntitySetIds, + principals = principals + ) + + return participantNeighbors.mapValues { + it.value.map { entityDetails -> getEntity(entityDetails.neighborDetails.get(), it.key, participants) } + }.values.flatten() + } + + private fun getEntity( + entity: Map>, + participant_ek_id: UUID, + participants: Map + ): PreProcessedEntity { + val participant = participants.getValue(participant_ek_id) + + return PreProcessedEntity( + study_id = participant.study_id, + participant_id = participant.legacy_participant_id, + appLabel = getFirstValueOrNull(entity, TITLE_FQN), + packageName = getFirstValueOrNull(entity, FULL_NAME_FQN)!!, + datetimeStart = getFirstValueOrNull(entity, DATE_TIME_START_FQN)?.let { OffsetDateTime.parse(it) }, + datetimeEnd = getFirstValueOrNull(entity, DATE_TIME_END_FQN)?.let { OffsetDateTime.parse(it) }, + timezone = getFirstValueOrNull(entity, TIMEZONE_FQN)!!, + recordType = getFirstValueOrNull(entity, RECORD_TYPE_FQN)!!, + newPeriod = getFirstValueOrNull(entity, NEW_PERIOD_FQN).toBoolean(), + duration = getFirstValueOrNull(entity, DURATION_FQN)?.toDouble(), + warning = getFirstValueOrNull(entity, WARNING_FQN), + newApp = getFirstValueOrNull(entity, NEW_APP_FQN).toBoolean() + ) + } + + private fun getLegacyParticipantEntitySetIds(): Set { + return entitySetIds.filter { it.key.startsWith("chronicle_participants_") }.map { it.value }.toSet() + } + + private fun getParticipantNeighbors( + entityKeyIds: Set, + entitySetIds: Map, + participantEntitySetIds: Set, + principals: Set + ): Map> { + val preprocessedEntitySetId = entitySetIds.getValue(PRE_PROCESSED_ES) + val participatedInEntitySetId = entitySetIds.getValue(PARTICIPATED_IN_ES) + val recordedByEntitySetId = entitySetIds.getValue(RECORDED_BY_ES) + + val filter = EntityNeighborsFilter( + entityKeyIds, + Optional.of(setOf(preprocessedEntitySetId)), + Optional.empty(), + Optional.of(setOf(recordedByEntitySetId, participatedInEntitySetId)) + ) + return searchService.executeEntityNeighborSearch( + participantEntitySetIds, + PagedNeighborRequest(filter), + principals + ).neighbors + } + + + // Returns participants in an org + private fun getOrgParticipants( + participantEntitySetIds: Set + ): Set { + return dataQueryService.getEntitiesWithPropertyTypeFqns( + participantEntitySetIds.associateWith { Optional.empty() }, + entitySetService.getPropertyTypesOfEntitySets(participantEntitySetIds), + mapOf(), + setOf(), + Optional.empty(), + false + ).mapValues { getParticipantEntity(it.key, it.value) }.values.filter { it.participantId != null }.toSet() + } + + + private fun getParticipantEntity(entityKeyId: UUID, entity: Map>): Participant { + val participantId = getFirstValueOrNull(entity, PERSON_FQN) + return Participant( + id = entityKeyId, + participantId = participantId + ) + } + + private fun getFirstValueOrNull(entity: Map>, fqn: FullQualifiedName): String? { + entity[fqn]?.iterator()?.let { + if (it.hasNext()) return it.next().toString() + } + return null + } + + + private fun getOrgEntitySetNames(orgId: UUID): Map { + val entitySetNameByTemplateName = when (orgId) { + LEGACY_ORG_ID -> mapOf( + PRE_PROCESSED_ES to LEGACY_PREPROCESSED_ES, + PARTICIPATED_IN_ES to LEGACY_PARTICIPATED_IN_ES, + STUDIES_ES to LEGACY_STUDIES_ES, + RECORDED_BY_ES to LEGACY_RECORDED_BY_ES + ) + else -> { + val orgIdToStr = orgId.toString().replace("-", "") + mapOf( + PRE_PROCESSED_ES to "$DATA_COLLECTION_APP_ES_PREFIX${orgIdToStr}_$PRE_PROCESSED_TEMPLATE", + PARTICIPATED_IN_ES to "$DATA_COLLECTION_APP_ES_PREFIX${orgIdToStr}_$PARTICIPATED_IN_TEMPLATE", + RECORDED_BY_ES to "$DATA_COLLECTION_APP_ES_PREFIX${orgIdToStr}_$RECORDED_BY_TEMPLATE", + PARTICIPANTS_ES to "$CHRONICLE_APP_ES_PREFIX${orgIdToStr}_${PARTICIPANTS_TEMPLATE}", + STUDIES_ES to "$CHRONICLE_APP_ES_PREFIX${orgIdToStr}_${STUDIES_TEMPLATE}E", + ) + } + } + + return entitySetNameByTemplateName.mapValues { entitySetIds.getValue(it.value) } + } + + private fun getChronicleSuperUserPrincipals(): Set { + val securablePrincipal = principalService.getSecurablePrincipal(SUPER_USER_PRINCIPAL_ID) + return principalService.getAllPrincipals(securablePrincipal).map { it.principal }.toSet() + Principal(PrincipalType.USER, SUPER_USER_PRINCIPAL_ID) + } +} + +private data class Study( + val studyEntityKeyId: UUID, + val studyId: UUID, + val title: String? +) + +data class Participant( + val id: UUID, + val participantId: String?, +) + +data class PreProcessedEntity( + val study_id: UUID, + val participant_id: String, + val appLabel: String?, + val datetimeStart: OffsetDateTime?, + val datetimeEnd: OffsetDateTime?, + val packageName: String, + val timezone: String, + val recordType: String, + val newPeriod: Boolean, + val newApp: Boolean, + val duration: Double?, + val warning: String? +) +data class ParticipantExport( + val participant_es_id: UUID, + val participant_ek_id: UUID, + val study_es_id: UUID, + val study_ek_id: UUID, + val organization_id: UUID, + val legacy_participant_id: String, + val study_id: UUID +) \ No newline at end of file From 5ec9461edba5b9a17fed11942621b93fb50f59a5 Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Sun, 6 Mar 2022 09:08:53 -0800 Subject: [PATCH 68/99] fix sql --- .../openlattice/mechanic/upgrades/MigratePreprocessedData.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt index 9df7f0ad..5937a8b6 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt @@ -108,7 +108,7 @@ class MigratePreprocessedData( $NEW_PERIOD boolean, $NEW_APP boolean, $DURATION double, - $WARNING text + $WARNING text, PRIMARY_KEY($PARTICIPANT_ID, $STUDY_ID, $DATE_TIME_START, $APP_PACKAGE_NAME) ) """.trimIndent() From fba0e09d4984d3262051a4dc8e764431195e58c7 Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Sun, 6 Mar 2022 09:51:46 -0800 Subject: [PATCH 69/99] fix --- .../openlattice/mechanic/upgrades/MigratePreprocessedData.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt index 5937a8b6..86fb5a01 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt @@ -109,7 +109,7 @@ class MigratePreprocessedData( $NEW_APP boolean, $DURATION double, $WARNING text, - PRIMARY_KEY($PARTICIPANT_ID, $STUDY_ID, $DATE_TIME_START, $APP_PACKAGE_NAME) + PRIMARY KEY ($PARTICIPANT_ID, $STUDY_ID, $DATE_TIME_START, $APP_PACKAGE_NAME) ) """.trimIndent() From a4cfe94301da458c63f6133ed56db734a4e6727b Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Sun, 6 Mar 2022 10:27:11 -0800 Subject: [PATCH 70/99] type double --- .../openlattice/mechanic/upgrades/MigratePreprocessedData.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt index 86fb5a01..6cbd1b96 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt @@ -107,7 +107,7 @@ class MigratePreprocessedData( $RECORD_TYPE text not null, $NEW_PERIOD boolean, $NEW_APP boolean, - $DURATION double, + $DURATION DOUBLE PRECISION , $WARNING text, PRIMARY KEY ($PARTICIPANT_ID, $STUDY_ID, $DATE_TIME_START, $APP_PACKAGE_NAME) ) From c688aa213b0c80f91aba7ebe2c73b148453e47a7 Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Sun, 6 Mar 2022 10:42:06 -0800 Subject: [PATCH 71/99] nullable --- .../mechanic/upgrades/MigratePreprocessedData.kt | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt index 6cbd1b96..063441b4 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt @@ -216,8 +216,7 @@ class MigratePreprocessedData( private fun participant(rs: ResultSet): ParticipantExport { return ParticipantExport( participant_ek_id = rs.getObject("participant_ek_id", UUID::class.java), - study_id = rs.getObject("study_id", UUID::class.java), - organization_id = rs.getObject("organization_id", UUID::class.java), + legacy_study_id = rs.getObject("legacy_study_id", UUID::class.java), study_ek_id = rs.getObject("study_ek_id", UUID::class.java), study_es_id = rs.getObject("study_es_id", UUID::class.java), legacy_participant_id = rs.getString("legacy_participant_id"), @@ -259,7 +258,7 @@ class MigratePreprocessedData( val participant = participants.getValue(participant_ek_id) return PreProcessedEntity( - study_id = participant.study_id, + study_id = participant.legacy_study_id, participant_id = participant.legacy_participant_id, appLabel = getFirstValueOrNull(entity, TITLE_FQN), packageName = getFirstValueOrNull(entity, FULL_NAME_FQN)!!, @@ -392,7 +391,6 @@ data class ParticipantExport( val participant_ek_id: UUID, val study_es_id: UUID, val study_ek_id: UUID, - val organization_id: UUID, + val legacy_study_id: UUID, val legacy_participant_id: String, - val study_id: UUID ) \ No newline at end of file From ee58cf119f773c3ffd1c1b2e49fe53d4d203a3d1 Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Sun, 6 Mar 2022 11:06:31 -0800 Subject: [PATCH 72/99] clean up --- .../mechanic/upgrades/MigratePreprocessedData.kt | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt index 063441b4..be3789b6 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt @@ -217,10 +217,7 @@ class MigratePreprocessedData( return ParticipantExport( participant_ek_id = rs.getObject("participant_ek_id", UUID::class.java), legacy_study_id = rs.getObject("legacy_study_id", UUID::class.java), - study_ek_id = rs.getObject("study_ek_id", UUID::class.java), - study_es_id = rs.getObject("study_es_id", UUID::class.java), legacy_participant_id = rs.getString("legacy_participant_id"), - participant_es_id = rs.getObject("participant_es_id", UUID::class.java) ) } @@ -232,14 +229,14 @@ class MigratePreprocessedData( LEGACY_ORG_ID -> getLegacyParticipantEntitySetIds() else -> setOf(orgEntitySetIds.getValue(PARTICIPANTS_ES)) } - val orgParticipants: Set = getOrgParticipants(participantEntitySetIds) + val orgParticipants: Set = getOrgParticipants(participantEntitySetIds) if (orgParticipants.isEmpty()) { logger.info("No participants found. Skipping org") return listOf() } val participantNeighbors: Map> = getParticipantNeighbors( - entityKeyIds = orgParticipants.map { it.id }.toSet(), + entityKeyIds = orgParticipants, entitySetIds = orgEntitySetIds, participantEntitySetIds = participantEntitySetIds, principals = principals @@ -304,7 +301,7 @@ class MigratePreprocessedData( // Returns participants in an org private fun getOrgParticipants( participantEntitySetIds: Set - ): Set { + ): Set { return dataQueryService.getEntitiesWithPropertyTypeFqns( participantEntitySetIds.associateWith { Optional.empty() }, entitySetService.getPropertyTypesOfEntitySets(participantEntitySetIds), @@ -312,7 +309,7 @@ class MigratePreprocessedData( setOf(), Optional.empty(), false - ).mapValues { getParticipantEntity(it.key, it.value) }.values.filter { it.participantId != null }.toSet() + ).keys } @@ -387,10 +384,7 @@ data class PreProcessedEntity( val warning: String? ) data class ParticipantExport( - val participant_es_id: UUID, val participant_ek_id: UUID, - val study_es_id: UUID, - val study_ek_id: UUID, val legacy_study_id: UUID, val legacy_participant_id: String, ) \ No newline at end of file From 9e27d88af789ea9e3d17999ab6ac4870da4ed1ed Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Sun, 6 Mar 2022 11:29:50 -0800 Subject: [PATCH 73/99] whattt --- .../upgrades/MigratePreprocessedData.kt | 36 ++++--------------- 1 file changed, 7 insertions(+), 29 deletions(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt index be3789b6..94235367 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt @@ -56,7 +56,6 @@ class MigratePreprocessedData( // collection template names private const val PRE_PROCESSED_TEMPLATE = "preprocesseddata"; - private const val STUDIES_TEMPLATE = "studies" private const val PARTICIPATED_IN_TEMPLATE = "participatedin" private const val PARTICIPANTS_TEMPLATE = "participants" private const val RECORDED_BY_TEMPLATE = "recordedby" @@ -64,7 +63,6 @@ class MigratePreprocessedData( // entity sets lookup name private const val RECORDED_BY_ES = "recordedBy" private const val PRE_PROCESSED_ES = "preprocessed" - private const val STUDIES_ES = "studies" private const val PARTICIPATED_IN_ES = "participatedIn" private const val PARTICIPANTS_ES = "participants" @@ -76,11 +74,11 @@ class MigratePreprocessedData( private const val DATE_TIME_END = "datetime_end" private const val APP_PACKAGE_NAME = "app_package_name" private const val TIMEZONE = "timezone" - private const val RECORD_TYPE ="record_type" - private const val NEW_PERIOD ="new_period" + private const val RECORD_TYPE = "record_type" + private const val NEW_PERIOD = "new_period" private const val NEW_APP = "new_app" private const val DURATION = "duration_seconds" - private const val WARNING ="warning" + private const val WARNING = "warning" private val PERSON_FQN = FullQualifiedName("nc.SubjectIdentification") private val TITLE_FQN = FullQualifiedName("ol.title") @@ -225,7 +223,7 @@ class MigratePreprocessedData( logger.info("getting preprocessed data entities for org $orgId") val orgEntitySetIds = getOrgEntitySetNames(orgId) - val participantEntitySetIds = when(orgId) { + val participantEntitySetIds = when (orgId) { LEGACY_ORG_ID -> getLegacyParticipantEntitySetIds() else -> setOf(orgEntitySetIds.getValue(PARTICIPANTS_ES)) } @@ -260,7 +258,7 @@ class MigratePreprocessedData( appLabel = getFirstValueOrNull(entity, TITLE_FQN), packageName = getFirstValueOrNull(entity, FULL_NAME_FQN)!!, datetimeStart = getFirstValueOrNull(entity, DATE_TIME_START_FQN)?.let { OffsetDateTime.parse(it) }, - datetimeEnd = getFirstValueOrNull(entity, DATE_TIME_END_FQN)?.let { OffsetDateTime.parse(it) }, + datetimeEnd = getFirstValueOrNull(entity, DATE_TIME_END_FQN)?.let { OffsetDateTime.parse(it) }, timezone = getFirstValueOrNull(entity, TIMEZONE_FQN)!!, recordType = getFirstValueOrNull(entity, RECORD_TYPE_FQN)!!, newPeriod = getFirstValueOrNull(entity, NEW_PERIOD_FQN).toBoolean(), @@ -313,14 +311,6 @@ class MigratePreprocessedData( } - private fun getParticipantEntity(entityKeyId: UUID, entity: Map>): Participant { - val participantId = getFirstValueOrNull(entity, PERSON_FQN) - return Participant( - id = entityKeyId, - participantId = participantId - ) - } - private fun getFirstValueOrNull(entity: Map>, fqn: FullQualifiedName): String? { entity[fqn]?.iterator()?.let { if (it.hasNext()) return it.next().toString() @@ -334,17 +324,15 @@ class MigratePreprocessedData( LEGACY_ORG_ID -> mapOf( PRE_PROCESSED_ES to LEGACY_PREPROCESSED_ES, PARTICIPATED_IN_ES to LEGACY_PARTICIPATED_IN_ES, - STUDIES_ES to LEGACY_STUDIES_ES, RECORDED_BY_ES to LEGACY_RECORDED_BY_ES ) else -> { val orgIdToStr = orgId.toString().replace("-", "") mapOf( PRE_PROCESSED_ES to "$DATA_COLLECTION_APP_ES_PREFIX${orgIdToStr}_$PRE_PROCESSED_TEMPLATE", - PARTICIPATED_IN_ES to "$DATA_COLLECTION_APP_ES_PREFIX${orgIdToStr}_$PARTICIPATED_IN_TEMPLATE", RECORDED_BY_ES to "$DATA_COLLECTION_APP_ES_PREFIX${orgIdToStr}_$RECORDED_BY_TEMPLATE", PARTICIPANTS_ES to "$CHRONICLE_APP_ES_PREFIX${orgIdToStr}_${PARTICIPANTS_TEMPLATE}", - STUDIES_ES to "$CHRONICLE_APP_ES_PREFIX${orgIdToStr}_${STUDIES_TEMPLATE}E", + PARTICIPATED_IN_ES to "$CHRONICLE_APP_ES_PREFIX${orgIdToStr}_$PARTICIPATED_IN_TEMPLATE", ) } } @@ -358,17 +346,6 @@ class MigratePreprocessedData( } } -private data class Study( - val studyEntityKeyId: UUID, - val studyId: UUID, - val title: String? -) - -data class Participant( - val id: UUID, - val participantId: String?, -) - data class PreProcessedEntity( val study_id: UUID, val participant_id: String, @@ -383,6 +360,7 @@ data class PreProcessedEntity( val duration: Double?, val warning: String? ) + data class ParticipantExport( val participant_ek_id: UUID, val legacy_study_id: UUID, From 14efcfff4e2072ba0e1768a82abd92f3b5af2f14 Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Sun, 6 Mar 2022 11:48:15 -0800 Subject: [PATCH 74/99] more npe --- .../mechanic/upgrades/MigratePreprocessedData.kt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt index 94235367..688f8706 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt @@ -259,8 +259,8 @@ class MigratePreprocessedData( packageName = getFirstValueOrNull(entity, FULL_NAME_FQN)!!, datetimeStart = getFirstValueOrNull(entity, DATE_TIME_START_FQN)?.let { OffsetDateTime.parse(it) }, datetimeEnd = getFirstValueOrNull(entity, DATE_TIME_END_FQN)?.let { OffsetDateTime.parse(it) }, - timezone = getFirstValueOrNull(entity, TIMEZONE_FQN)!!, - recordType = getFirstValueOrNull(entity, RECORD_TYPE_FQN)!!, + timezone = getFirstValueOrNull(entity, TIMEZONE_FQN), + recordType = getFirstValueOrNull(entity, RECORD_TYPE_FQN), newPeriod = getFirstValueOrNull(entity, NEW_PERIOD_FQN).toBoolean(), duration = getFirstValueOrNull(entity, DURATION_FQN)?.toDouble(), warning = getFirstValueOrNull(entity, WARNING_FQN), @@ -353,8 +353,8 @@ data class PreProcessedEntity( val datetimeStart: OffsetDateTime?, val datetimeEnd: OffsetDateTime?, val packageName: String, - val timezone: String, - val recordType: String, + val timezone: String?, + val recordType: String?, val newPeriod: Boolean, val newApp: Boolean, val duration: Double?, From 9c6d73bcb488a590eca79223b965d4fcf21af39f Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Sun, 6 Mar 2022 12:31:36 -0800 Subject: [PATCH 75/99] fix --- .../mechanic/upgrades/MigratePreprocessedData.kt | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt index 688f8706..947f82c1 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt @@ -98,16 +98,15 @@ class MigratePreprocessedData( $STUDY_ID uuid not null, $PARTICIPANT_ID text not null, $APP_LABEL text, - $APP_PACKAGE_NAME text not null, + $APP_PACKAGE_NAME text, $DATE_TIME_START timestamp with time zone, $DATE_TIME_END timestamp with time zone, - $TIMEZONE text not null, - $RECORD_TYPE text not null, + $TIMEZONE text, + $RECORD_TYPE text, $NEW_PERIOD boolean, $NEW_APP boolean, $DURATION DOUBLE PRECISION , - $WARNING text, - PRIMARY KEY ($PARTICIPANT_ID, $STUDY_ID, $DATE_TIME_START, $APP_PACKAGE_NAME) + $WARNING text ) """.trimIndent() @@ -256,7 +255,7 @@ class MigratePreprocessedData( study_id = participant.legacy_study_id, participant_id = participant.legacy_participant_id, appLabel = getFirstValueOrNull(entity, TITLE_FQN), - packageName = getFirstValueOrNull(entity, FULL_NAME_FQN)!!, + packageName = getFirstValueOrNull(entity, FULL_NAME_FQN), datetimeStart = getFirstValueOrNull(entity, DATE_TIME_START_FQN)?.let { OffsetDateTime.parse(it) }, datetimeEnd = getFirstValueOrNull(entity, DATE_TIME_END_FQN)?.let { OffsetDateTime.parse(it) }, timezone = getFirstValueOrNull(entity, TIMEZONE_FQN), @@ -352,7 +351,7 @@ data class PreProcessedEntity( val appLabel: String?, val datetimeStart: OffsetDateTime?, val datetimeEnd: OffsetDateTime?, - val packageName: String, + val packageName: String?, val timezone: String?, val recordType: String?, val newPeriod: Boolean, From d58694a848844cc9f1fe57c30dd73231e5638d5d Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Sun, 6 Mar 2022 14:28:22 -0800 Subject: [PATCH 76/99] filter out nulls --- .../mechanic/upgrades/MigratePreprocessedData.kt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt index 947f82c1..938b34fb 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt @@ -241,7 +241,7 @@ class MigratePreprocessedData( return participantNeighbors.mapValues { it.value.map { entityDetails -> getEntity(entityDetails.neighborDetails.get(), it.key, participants) } - }.values.flatten() + }.values.flatten().filter { it.study_id != null || it.participant_id != null } } private fun getEntity( @@ -249,11 +249,11 @@ class MigratePreprocessedData( participant_ek_id: UUID, participants: Map ): PreProcessedEntity { - val participant = participants.getValue(participant_ek_id) + val participant = participants[participant_ek_id] return PreProcessedEntity( - study_id = participant.legacy_study_id, - participant_id = participant.legacy_participant_id, + study_id = participant?.legacy_study_id, + participant_id = participant?.legacy_participant_id, appLabel = getFirstValueOrNull(entity, TITLE_FQN), packageName = getFirstValueOrNull(entity, FULL_NAME_FQN), datetimeStart = getFirstValueOrNull(entity, DATE_TIME_START_FQN)?.let { OffsetDateTime.parse(it) }, @@ -346,8 +346,8 @@ class MigratePreprocessedData( } data class PreProcessedEntity( - val study_id: UUID, - val participant_id: String, + val study_id: UUID?, + val participant_id: String?, val appLabel: String?, val datetimeStart: OffsetDateTime?, val datetimeEnd: OffsetDateTime?, From a1c4d4eea7b8c9d1eb8be4c677bcd11123258d78 Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Wed, 9 Mar 2022 10:17:06 -0800 Subject: [PATCH 77/99] rename --- .../kotlin/com/openlattice/mechanic/pods/MechanicUpgradePod.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/pods/MechanicUpgradePod.kt b/src/main/kotlin/com/openlattice/mechanic/pods/MechanicUpgradePod.kt index f28c001c..783d54fe 100644 --- a/src/main/kotlin/com/openlattice/mechanic/pods/MechanicUpgradePod.kt +++ b/src/main/kotlin/com/openlattice/mechanic/pods/MechanicUpgradePod.kt @@ -397,7 +397,7 @@ class MechanicUpgradePod { } @Bean - fun migrateTImeUseDiarySummarizedData(): MigrateTimeUseDiarySummarizedData { + fun migrateTimeUseDiarySummarizedData(): MigrateTimeUseDiarySummarizedData { return MigrateTimeUseDiarySummarizedData( toolbox, rhizomeConfiguration, From c41e97a1e5f26940f70be9382287f58edbfde235 Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Wed, 9 Mar 2022 10:45:58 -0800 Subject: [PATCH 78/99] fix conflicts --- .../MigrateChronicleParticipantStats.kt | 11 +++++----- .../upgrades/MigrateOrgSettingsToStudies.kt | 6 ------ .../MigrateTimeUseDiarySubmissions.kt | 20 +++++-------------- .../MigrateTimeUseDiarySummarizedData.kt | 14 +++++-------- 4 files changed, 16 insertions(+), 35 deletions(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt index 7442f2d3..f99a6c88 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt @@ -344,7 +344,7 @@ class MigrateChronicleParticipantStats( organizationId = orgId, studyEntityKeyId = studyEntityKeyId, studyId = studies.getValue(studyEntityKeyId).studyId, - participantId = participantById.getValue(id).participantId, + participantId = participantById.getValue(id).participantId!!, androidFirstDate = androidStats.first, androidLastDate = androidStats.second, androidUniqueDates = androidStats.third, @@ -477,14 +477,14 @@ class MigrateChronicleParticipantStats( private fun getStudyEntity(studyEntityKeyId: UUID, entity: Map>): Study { val title = getFirstValueOrNull(entity, FULL_NAME_FQN) val studyId = getFirstUUIDOrNull(entity, STRING_ID_FQN) - return Study(studyEntityKeyId, studyId!!, title) + return Study(studyEntityKeyId, studyId!!, title = title) } } data class Participant( val studyEntityKeyId: UUID, val id: UUID, - val participantId: String, + val participantId: String?, ) private data class ParticipantStats( @@ -500,8 +500,9 @@ private data class ParticipantStats( val tudUniqueDates: Set = setOf() ) -private data class Study( +data class Study( val studyEntityKeyId: UUID, val studyId: UUID, - val title: String? + val settings: Map = mapOf(), + val title: String? = "" ) \ No newline at end of file diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateOrgSettingsToStudies.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateOrgSettingsToStudies.kt index eb5e97e5..764af5da 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateOrgSettingsToStudies.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateOrgSettingsToStudies.kt @@ -202,12 +202,6 @@ class MigrateOrgSettingsToStudies( } } -private data class Study( - val studyId: UUID, - val studyEntityKeyId: UUID, - val settings: Map -) - private enum class AppComponents { CHRONICLE, CHRONICLE_DATA_COLLECTION, diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateTimeUseDiarySubmissions.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateTimeUseDiarySubmissions.kt index 5ef374cb..e2f11412 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateTimeUseDiarySubmissions.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateTimeUseDiarySubmissions.kt @@ -389,7 +389,7 @@ class MigrateTimeUseDiarySubmissions( private fun getStudyEntity(studyEntityKeyId: UUID, entity: Map>): Study { val title = getFirstValueOrNull(entity, FULL_NAME_FQN) val studyId = getFirstUUIDOrNull(entity, STRING_ID_FQN) - return Study(studyEntityKeyId, studyId!!, title) + return Study(studyEntityKeyId, studyId!!, title = title) } // Returns a mapping from studyEntityKeyId to list of participants @@ -411,7 +411,8 @@ class MigrateTimeUseDiarySubmissions( val participantId = getFirstValueOrNull(entity, PERSON_FQN) return Participant( id = entityKeyId, - participantId = participantId + participantId = participantId, + studyEntityKeyId = entityKeyId ) } @@ -478,18 +479,7 @@ class MigrateTimeUseDiarySubmissions( } } -private data class Study( - val studyEntityKeyId: UUID, - val studyId: UUID, - val title: String? -) - -data class Participant( - val id: UUID, - val participantId: String?, -) - -data class ResponseEntity( +private data class ResponseEntity( val code: String?, val question: String?, val response: Set, @@ -497,7 +487,7 @@ data class ResponseEntity( val endDateTime: OffsetDateTime? ) -data class SubmissionEntity( +private data class SubmissionEntity( val orgId: UUID, val studyId: UUID, val submissionId: UUID, diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateTimeUseDiarySummarizedData.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateTimeUseDiarySummarizedData.kt index 1db51be5..8b993971 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateTimeUseDiarySummarizedData.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateTimeUseDiarySummarizedData.kt @@ -1,10 +1,7 @@ package com.openlattice.mechanic.upgrades -import com.fasterxml.jackson.databind.json.JsonMapper import com.geekbeast.mappers.mappers.ObjectMappers import com.geekbeast.rhizome.configuration.RhizomeConfiguration -import com.geekbeast.util.log -import com.openlattice.authorization.AuthorizationManager import com.openlattice.authorization.Principal import com.openlattice.authorization.PrincipalType import com.openlattice.data.requests.NeighborEntityDetails @@ -20,7 +17,6 @@ import com.zaxxer.hikari.HikariConfig import com.zaxxer.hikari.HikariDataSource import org.apache.olingo.commons.api.edm.FullQualifiedName import org.slf4j.LoggerFactory -import java.time.OffsetDateTime import java.util.* /** @@ -91,7 +87,7 @@ class MigrateTimeUseDiarySummarizedData( return Version.V2021_07_23.value } - private fun writeEntitiesToTable(entities: Set): Int { + private fun writeEntitiesToTable(entities: Set): Int { val hds = getHikariDataSource() return hds.connection.use { connection -> val wc = connection.prepareStatement(INSERT_INTO_TABLE_SQL).use { ps -> @@ -107,7 +103,7 @@ class MigrateTimeUseDiarySummarizedData( } - private fun getEntitiesForOrg(orgId: UUID, principals: Set): Set { + private fun getEntitiesForOrg(orgId: UUID, principals: Set): Set { logger.info("processing org $orgId") val entitySets = getOrgEntitySetNames(orgId) logger.info("entity sets: $entitySets") @@ -159,7 +155,7 @@ class MigrateTimeUseDiarySummarizedData( return null } - private fun getSummaryEntityForSubmission(submissionId: UUID, neighbors: List): SubmissionEntity { + private fun getSummaryEntityForSubmission(submissionId: UUID, neighbors: List): SummarizedDataSubmissionEntity { val values = neighbors.map { val entity = it.neighborDetails.get() SummarizedEntity( @@ -168,7 +164,7 @@ class MigrateTimeUseDiarySummarizedData( ) }.filter { it.value != null && it.variable != null }.toSet() - return SubmissionEntity( + return SummarizedDataSubmissionEntity( submissionId = submissionId, entities = values, ) @@ -204,7 +200,7 @@ private data class SummarizedEntity( val value: String? ) -private data class SubmissionEntity( +private data class SummarizedDataSubmissionEntity( val submissionId: UUID, val entities: Set, ) From 89998eec774cb84444827b20e956e5301eef72b5 Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Wed, 9 Mar 2022 11:51:56 -0800 Subject: [PATCH 79/99] rename --- .../com/openlattice/mechanic/pods/MechanicUpgradePod.kt | 4 ++-- ...ppUsageSurveyMigration.kt => MigrateAppUsageSurveyData.kt} | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) rename src/main/kotlin/com/openlattice/mechanic/upgrades/{V3AppUsageSurveyMigration.kt => MigrateAppUsageSurveyData.kt} (99%) diff --git a/src/main/kotlin/com/openlattice/mechanic/pods/MechanicUpgradePod.kt b/src/main/kotlin/com/openlattice/mechanic/pods/MechanicUpgradePod.kt index a1b40028..66be3137 100644 --- a/src/main/kotlin/com/openlattice/mechanic/pods/MechanicUpgradePod.kt +++ b/src/main/kotlin/com/openlattice/mechanic/pods/MechanicUpgradePod.kt @@ -384,8 +384,8 @@ class MechanicUpgradePod { } @Bean - fun v3AppUsageSurveyMigration(): V3AppUsageSurveyMigration { - return V3AppUsageSurveyMigration( + fun migrateAppUsageSurveyData(): MigrateAppUsageSurveyData { + return MigrateAppUsageSurveyData( toolbox, rhizomeConfiguration, authorizationService(), diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/V3AppUsageSurveyMigration.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateAppUsageSurveyData.kt similarity index 99% rename from src/main/kotlin/com/openlattice/mechanic/upgrades/V3AppUsageSurveyMigration.kt rename to src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateAppUsageSurveyData.kt index a0ca352b..25da6ba1 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/V3AppUsageSurveyMigration.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateAppUsageSurveyData.kt @@ -25,7 +25,7 @@ import java.util.* /** * @author alfoncenzioka <alfonce@openlattice.com> */ -class V3AppUsageSurveyMigration( +class MigrateAppUsageSurveyData( toolbox: Toolbox, private val rhizomeConfiguration: RhizomeConfiguration, private val authorizationService: AuthorizationManager, @@ -34,7 +34,7 @@ class V3AppUsageSurveyMigration( private val dataQueryService: PostgresEntityDataQueryService, private val entitySetService: EntitySetManager ) : Upgrade { - private val logger = LoggerFactory.getLogger(V3AppUsageSurveyMigration::class.java) + private val logger = LoggerFactory.getLogger(MigrateAppUsageSurveyData::class.java) private val organizations = HazelcastMap.ORGANIZATIONS.getMap(toolbox.hazelcast) private val appConfigs = HazelcastMap.APP_CONFIGS.getMap(toolbox.hazelcast) From fbba4172a2ba5e191abdae4192b2e60fbb577389 Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Wed, 9 Mar 2022 12:56:10 -0800 Subject: [PATCH 80/99] rename tables --- .../mechanic/upgrades/MigrateAppUsageSurveyData.kt | 2 +- .../mechanic/upgrades/MigrateChronicleParticipantStats.kt | 2 +- .../mechanic/upgrades/MigrateChronicleSystemApps.kt | 5 +++-- .../mechanic/upgrades/MigrateOrgSettingsToStudies.kt | 2 +- .../mechanic/upgrades/MigrateTimeUseDiarySubmissions.kt | 2 +- .../mechanic/upgrades/MigrateTimeUseDiarySummarizedData.kt | 6 ++++-- 6 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateAppUsageSurveyData.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateAppUsageSurveyData.kt index 25da6ba1..4cd18fcb 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateAppUsageSurveyData.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateAppUsageSurveyData.kt @@ -96,7 +96,7 @@ class MigrateAppUsageSurveyData( private val APP_USAGE_SURVEY_COLUMNS = column_names.joinToString(",") { it } private val APP_USAGE_SURVEY_PARAMS = column_names.joinToString(",") { "?" } - private const val TABLE_NAME = "public.app_usage_survey" + private const val TABLE_NAME = "migrate_app_usage_survey" /** * PreparedStatement bind order diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt index f99a6c88..1e56aefd 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleParticipantStats.kt @@ -86,7 +86,7 @@ class MigrateChronicleParticipantStats( private val DATETIME_FQN = FullQualifiedName("ol.datetime") private val RECORDED_DATE_FQN = FullQualifiedName("ol.recordeddate") - private const val TABLE_NAME = "participant_stats" + private const val TABLE_NAME = "migrate_participant_stats" // column names private const val ORGANIZATION_IO = "organization_id" diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleSystemApps.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleSystemApps.kt index 14c37b6c..a34cc4a5 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleSystemApps.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleSystemApps.kt @@ -35,15 +35,16 @@ class MigrateChronicleSystemApps( private val FULL_NAME_FQN = FullQualifiedName("general.fullname") private val RECORD_TYPE_FQN = FullQualifiedName("ol.recordtype") + private const val TABLE_NAME = "migrate_system_apps" private val CREATE_SYSTEM_APPS_SQL = """ - CREATE TABLE IF NOT EXISTS public.system_apps( + CREATE TABLE IF NOT EXISTS $TABLE_NAME( app_package_name text NOT NULL, PRIMARY KEY (app_package_name) ) """.trimIndent() private val INSERT_SYSTEM_APPS_SQL = """ - INSERT INTO public.system_apps values(?) ON CONFLICT DO NOTHING + INSERT INTO $TABLE_NAME values(?) ON CONFLICT DO NOTHING """.trimIndent() } diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateOrgSettingsToStudies.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateOrgSettingsToStudies.kt index 764af5da..833a0de9 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateOrgSettingsToStudies.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateOrgSettingsToStudies.kt @@ -57,7 +57,7 @@ class MigrateOrgSettingsToStudies( private const val LEGACY_STUDY_EK_ID = "v2_study_ekid" private const val SETTINGS = "settings" - private const val STUDY_SETTINGS_TABLE = "study_settings" + private const val STUDY_SETTINGS_TABLE = "migrate_study_settings" private val CREATE_TABLE_QUERY = """ CREATE TABLE IF NOT EXISTS $STUDY_SETTINGS_TABLE( diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateTimeUseDiarySubmissions.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateTimeUseDiarySubmissions.kt index e2f11412..86e1cf56 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateTimeUseDiarySubmissions.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateTimeUseDiarySubmissions.kt @@ -92,7 +92,7 @@ class MigrateTimeUseDiarySubmissions( private const val SUBMISSION_ID = "submission_id" //not unique for each row private const val SUBMISSION_DATE = "submission_date" - private const val TABLE_NAME = "time_use_diary_v2" + private const val TABLE_NAME = "migrate_time_use_diary" private val COLUMNS = linkedSetOf( STUDY_ID, diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateTimeUseDiarySummarizedData.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateTimeUseDiarySummarizedData.kt index 8b993971..6ae485f6 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateTimeUseDiarySummarizedData.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateTimeUseDiarySummarizedData.kt @@ -53,8 +53,10 @@ class MigrateTimeUseDiarySummarizedData( private val VARIABLE_FQN = FullQualifiedName("ol.variable") private val VALUES_FQN = FullQualifiedName("ol.values") + private const val TABLE_NAME = "migrate_time_use_diary_summary" + private val CREATE_TABLE_SQL = """ - CREATE TABLE IF NOT EXISTS time_use_diary_summary( + CREATE TABLE IF NOT EXISTS $TABLE_NAME( submission_id uuid not null, data jsonb not null, PRIMARY KEY (submission_id) @@ -62,7 +64,7 @@ class MigrateTimeUseDiarySummarizedData( """.trimIndent() private val INSERT_INTO_TABLE_SQL = """ - INSERT INTO time_use_diary_summary values(?, ?::jsonb) + INSERT INTO $TABLE_NAME values(?, ?::jsonb) """.trimIndent() } From da96e634c8bb5930f26a20c9b29dc2627b05be95 Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Wed, 9 Mar 2022 14:16:48 -0800 Subject: [PATCH 81/99] table DNE --- .../openlattice/mechanic/upgrades/MigrateAppUsageSurveyData.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateAppUsageSurveyData.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateAppUsageSurveyData.kt index 4cd18fcb..67b385fe 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateAppUsageSurveyData.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateAppUsageSurveyData.kt @@ -111,7 +111,7 @@ class MigrateAppUsageSurveyData( * 9) users */ private val INSERT_INTO_APP_USAGE_SQL = """ - INSERT INTO app_usage_survey($APP_USAGE_SURVEY_COLUMNS) values ($APP_USAGE_SURVEY_PARAMS) + INSERT INTO $TABLE_NAME($APP_USAGE_SURVEY_COLUMNS) values ($APP_USAGE_SURVEY_PARAMS) ON CONFLICT DO NOTHING """.trimIndent() From 62a4c20f2d37667581b5b2e554960905b7eaf755 Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Wed, 9 Mar 2022 17:39:01 -0800 Subject: [PATCH 82/99] write users to chronicle --- .../mechanic/pods/MechanicUpgradePod.kt | 2 +- .../mechanic/upgrades/ExportOrganizationMembers.kt | 14 +++++++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/pods/MechanicUpgradePod.kt b/src/main/kotlin/com/openlattice/mechanic/pods/MechanicUpgradePod.kt index 66be3137..a5528c99 100644 --- a/src/main/kotlin/com/openlattice/mechanic/pods/MechanicUpgradePod.kt +++ b/src/main/kotlin/com/openlattice/mechanic/pods/MechanicUpgradePod.kt @@ -370,7 +370,7 @@ class MechanicUpgradePod { @Bean fun exportOrganizationMembers(): ExportOrganizationMembers { - return ExportOrganizationMembers(toolbox, hikariDataSource, principalService(), hazelcastInstance) + return ExportOrganizationMembers(toolbox, hikariDataSource, principalService(), hazelcastInstance, rhizomeConfiguration) } @Bean diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/ExportOrganizationMembers.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/ExportOrganizationMembers.kt index 45df393a..9d8ff6ae 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/ExportOrganizationMembers.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/ExportOrganizationMembers.kt @@ -1,9 +1,11 @@ package com.openlattice.mechanic.upgrades +import com.geekbeast.rhizome.configuration.RhizomeConfiguration import com.hazelcast.core.HazelcastInstance import com.openlattice.hazelcast.HazelcastMap import com.openlattice.mechanic.Toolbox import com.openlattice.organizations.roles.SecurePrincipalsManager +import com.zaxxer.hikari.HikariConfig import com.zaxxer.hikari.HikariDataSource import org.slf4j.LoggerFactory @@ -15,12 +17,13 @@ class ExportOrganizationMembers( private val toolbox: Toolbox, private val hds: HikariDataSource, private val principalService: SecurePrincipalsManager, - hazelcast :HazelcastInstance + hazelcast :HazelcastInstance, + private val rhizomeConfiguration: RhizomeConfiguration ) : Upgrade { companion object { private const val USERS_EXPORT_TABLE_NAME = "users_export" const val USERS_EXPORT_TABLE = """ - CREATE TABLE $USERS_EXPORT_TABLE_NAME ( + CREATE TABLE IF NOT EXISTS $USERS_EXPORT_TABLE_NAME ( organization_id uuid, principal_id text NOT NULL, principal_email text, @@ -36,11 +39,16 @@ class ExportOrganizationMembers( private val organizations = HazelcastMap.ORGANIZATIONS.getMap(hazelcast) private val users = HazelcastMap.USERS.getMap(hazelcast) + private fun getDatasource(): HikariDataSource { + val (hikariConfiguration) = rhizomeConfiguration.datasourceConfigurations["chronicle"]!! + val hc = HikariConfig(hikariConfiguration) + return HikariDataSource(hc) + } override fun upgrade(): Boolean { val organizationsIds = organizations.keys.toMutableSet() val members = principalService.getOrganizationMembers(organizationsIds) - hds.connection.use { connection -> + getDatasource().connection.use { connection -> connection.createStatement().use { stmt -> stmt.execute(USERS_EXPORT_TABLE) } connection.prepareStatement(INSERT_USER_SQL).use { ps -> members.forEach { (orgId, orgMembers) -> From 2f90771bcccc84f05c53e3041d2701e5c6caa39b Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Wed, 9 Mar 2022 18:34:28 -0800 Subject: [PATCH 83/99] chronicle super user principal --- .../mechanic/upgrades/V3StudyMigrationUpgrade.kt | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/V3StudyMigrationUpgrade.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/V3StudyMigrationUpgrade.kt index 2b22c43d..789df70e 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/V3StudyMigrationUpgrade.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/V3StudyMigrationUpgrade.kt @@ -2,6 +2,8 @@ package com.openlattice.mechanic.upgrades import com.geekbeast.rhizome.configuration.RhizomeConfiguration import com.hazelcast.query.Predicates +import com.openlattice.authorization.Principal +import com.openlattice.authorization.PrincipalType import com.openlattice.data.storage.MetadataOption import com.openlattice.data.storage.postgres.PostgresEntityDataQueryService import com.openlattice.edm.EdmConstants.Companion.LAST_WRITE_FQN @@ -32,6 +34,7 @@ class V3StudyMigrationUpgrade( private val searchService: SearchService ): Upgrade { + private val SUPER_USER_PRINCIPAL_ID = "auth0|5ae9026c04eb0b243f1d2bb6" private val logger = LoggerFactory.getLogger(V3StudyMigrationUpgrade::class.java) private val propertyTypes = HazelcastMap.PROPERTY_TYPES.getMap(toolbox.hazelcast) @@ -254,8 +257,8 @@ class V3StudyMigrationUpgrade( val filter = EntityNeighborsFilter(setOf(studyEkid), Optional.of(orgMaybeParticipantEntitySetIds), Optional.of(orgStudyEntitySetIds), Optional.empty()) - val chronicleSuperUserSecurablePrincipal = principalService.getSecurablePrincipal("") - val chronicleSuperUserPrincipals = principalService.getAllPrincipals(chronicleSuperUserSecurablePrincipal).map { it.principal }.toSet() + val chronicleSuperUserSecurablePrincipal = principalService.getSecurablePrincipal(SUPER_USER_PRINCIPAL_ID) + val chronicleSuperUserPrincipals = principalService.getAllPrincipals(chronicleSuperUserSecurablePrincipal).map { it.principal }.toSet() + Principal(PrincipalType.USER, SUPER_USER_PRINCIPAL_ID) logger.info("chronicle super user principals ${chronicleSuperUserPrincipals.size} $chronicleSuperUserPrincipals") // get all participants for the study From 1f2fa8d4fdf61970ae0f21e07d6be7601cafbd5b Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Wed, 9 Mar 2022 19:08:11 -0800 Subject: [PATCH 84/99] conflict --- .../mechanic/upgrades/MigrateTimeUseDiarySummarizedData.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateTimeUseDiarySummarizedData.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateTimeUseDiarySummarizedData.kt index 6ae485f6..d30cada5 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateTimeUseDiarySummarizedData.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateTimeUseDiarySummarizedData.kt @@ -64,7 +64,7 @@ class MigrateTimeUseDiarySummarizedData( """.trimIndent() private val INSERT_INTO_TABLE_SQL = """ - INSERT INTO $TABLE_NAME values(?, ?::jsonb) + INSERT INTO $TABLE_NAME values(?, ?::jsonb) ON CONFLICT DO NOTHING """.trimIndent() } From 3ad03cf300dca5bdb61aa04d07a37ba192bc4b47 Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Thu, 10 Mar 2022 03:07:43 -0800 Subject: [PATCH 85/99] batch processing --- .../upgrades/MigratePreprocessedData.kt | 44 ++++++++++++------- 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt index 938b34fb..4e2f4dff 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt @@ -156,15 +156,15 @@ class MigratePreprocessedData( val participants = BasePostgresIterable( PreparedStatementHolderSupplier( getHikariDataSource(), - "SELECT * FROM participant_export" + "SELECT * FROM participants_export" ) {} ) { participant(it) - }.associateBy { it.participant_ek_id } + }.groupBy { it.organization_id } val orgIds = (appConfigs.keys.filter { it.appId == DATA_COLLECTION_APP_ID }.map { it.organizationId } + LEGACY_ORG_ID).toSet() val principals = getChronicleSuperUserPrincipals() - val entitiesToWrite = orgIds.associateWith { getEntitiesForOrg(it, participants, principals) }.values.flatten() + val entitiesToWrite = orgIds.associateWith { getEntitiesForOrg(it, participants.getValue(it).associateBy { participant -> participant.participant_ek_id }, principals) }.values.flatten() val written = writeEntities(entitiesToWrite) logger.info("Exported $written entities to preprocessed table") @@ -215,6 +215,7 @@ class MigratePreprocessedData( participant_ek_id = rs.getObject("participant_ek_id", UUID::class.java), legacy_study_id = rs.getObject("legacy_study_id", UUID::class.java), legacy_participant_id = rs.getString("legacy_participant_id"), + organization_id = rs.getObject("organization_id", UUID::class.java) ) } @@ -223,25 +224,37 @@ class MigratePreprocessedData( val orgEntitySetIds = getOrgEntitySetNames(orgId) val participantEntitySetIds = when (orgId) { - LEGACY_ORG_ID -> getLegacyParticipantEntitySetIds() + LEGACY_ORG_ID -> { + val entitySetNames = participants.values.map { it.legacy_study_id }.map { studyId -> "chronicle_participants_$studyId" } + entitySetIds.filter { entitySetNames.contains(it.key) }.values.toSet() + } else -> setOf(orgEntitySetIds.getValue(PARTICIPANTS_ES)) } - val orgParticipants: Set = getOrgParticipants(participantEntitySetIds) - if (orgParticipants.isEmpty()) { + if (participants.isEmpty()) { logger.info("No participants found. Skipping org") return listOf() } - val participantNeighbors: Map> = getParticipantNeighbors( - entityKeyIds = orgParticipants, - entitySetIds = orgEntitySetIds, - participantEntitySetIds = participantEntitySetIds, - principals = principals - ) + val result: MutableList = mutableListOf() + val allIds = participants.keys.toMutableSet() + while (allIds.isNotEmpty()) { + val current = allIds.take(20).toSet() + logger.info("processing $current entity key ids. Remaining ${(allIds - current).size}") + val participantNeighbors: Map> = getParticipantNeighbors( + entityKeyIds = current, + entitySetIds = orgEntitySetIds, + participantEntitySetIds = participantEntitySetIds, + principals = principals + ) + val entities = participantNeighbors.mapValues { + it.value.map { entityDetails -> getEntity(entityDetails.neighborDetails.get(), it.key, participants) } + }.values.flatten().filter { it.study_id != null || it.participant_id != null } - return participantNeighbors.mapValues { - it.value.map { entityDetails -> getEntity(entityDetails.neighborDetails.get(), it.key, participants) } - }.values.flatten().filter { it.study_id != null || it.participant_id != null } + result.addAll(entities); + + allIds -= current + } + return result } private fun getEntity( @@ -364,4 +377,5 @@ data class ParticipantExport( val participant_ek_id: UUID, val legacy_study_id: UUID, val legacy_participant_id: String, + val organization_id: UUID ) \ No newline at end of file From a8e618a75fdef9a141c9a933277095ffcc324577 Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Thu, 10 Mar 2022 03:25:08 -0800 Subject: [PATCH 86/99] log --- .../openlattice/mechanic/upgrades/MigratePreprocessedData.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt index 4e2f4dff..4d7da61a 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt @@ -239,7 +239,7 @@ class MigratePreprocessedData( val allIds = participants.keys.toMutableSet() while (allIds.isNotEmpty()) { val current = allIds.take(20).toSet() - logger.info("processing $current entity key ids. Remaining ${(allIds - current).size}") + logger.info("processing ${current.size} entity key ids. Remaining ${(allIds - current).size}") val participantNeighbors: Map> = getParticipantNeighbors( entityKeyIds = current, entitySetIds = orgEntitySetIds, From 5964fa04281edea97aacc28a19eedb53df02e33d Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Thu, 10 Mar 2022 03:25:55 -0800 Subject: [PATCH 87/99] batch size --- .../openlattice/mechanic/upgrades/MigratePreprocessedData.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt index 4d7da61a..ee32a722 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt @@ -238,7 +238,7 @@ class MigratePreprocessedData( val result: MutableList = mutableListOf() val allIds = participants.keys.toMutableSet() while (allIds.isNotEmpty()) { - val current = allIds.take(20).toSet() + val current = allIds.take(50).toSet() logger.info("processing ${current.size} entity key ids. Remaining ${(allIds - current).size}") val participantNeighbors: Map> = getParticipantNeighbors( entityKeyIds = current, From 33fbd3116170a6f6357ebf198fbb0fb666c961e2 Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Thu, 10 Mar 2022 03:36:14 -0800 Subject: [PATCH 88/99] log --- .../openlattice/mechanic/upgrades/MigratePreprocessedData.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt index ee32a722..eb4d95ec 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt @@ -250,7 +250,8 @@ class MigratePreprocessedData( it.value.map { entityDetails -> getEntity(entityDetails.neighborDetails.get(), it.key, participants) } }.values.flatten().filter { it.study_id != null || it.participant_id != null } - result.addAll(entities); + logger.info("retrieved ${entities.size} entities") + result.addAll(entities) allIds -= current } From c4f9bd85110200c2fd44e145a65448dc57f397b1 Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Thu, 10 Mar 2022 03:37:49 -0800 Subject: [PATCH 89/99] log --- .../openlattice/mechanic/upgrades/MigratePreprocessedData.kt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt index eb4d95ec..df1e8870 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt @@ -239,7 +239,7 @@ class MigratePreprocessedData( val allIds = participants.keys.toMutableSet() while (allIds.isNotEmpty()) { val current = allIds.take(50).toSet() - logger.info("processing ${current.size} entity key ids. Remaining ${(allIds - current).size}") + logger.info("processing neighbors of ${current.size} participants. Remaining ${(allIds - current).size}") val participantNeighbors: Map> = getParticipantNeighbors( entityKeyIds = current, entitySetIds = orgEntitySetIds, @@ -250,11 +250,12 @@ class MigratePreprocessedData( it.value.map { entityDetails -> getEntity(entityDetails.neighborDetails.get(), it.key, participants) } }.values.flatten().filter { it.study_id != null || it.participant_id != null } - logger.info("retrieved ${entities.size} entities") + logger.info("retrieved ${entities.size} preprocessed entities") result.addAll(entities) allIds -= current } + logger.info("Total preprocessed entities retrieved for org: ${result.size}") return result } From 535ffbd2ff7cbdcfc76018be4e077918bc8b1423 Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Thu, 10 Mar 2022 04:01:09 -0800 Subject: [PATCH 90/99] filter out invalid configs --- .../mechanic/upgrades/MigratePreprocessedData.kt | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt index df1e8870..e625d588 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt @@ -37,6 +37,8 @@ class MigratePreprocessedData( private val entitySetIds: Map = HazelcastMap.ENTITY_SETS.getMap(toolbox.hazelcast).associate { it.value.name to it.key } private val appConfigs = HazelcastMap.APP_CONFIGS.getMap(toolbox.hazelcast) + private val organizations = HazelcastMap.ORGANIZATIONS.getMap(toolbox.hazelcast) + companion object { private val logger = LoggerFactory.getLogger(MigratePreprocessedData::class.java) @@ -164,7 +166,10 @@ class MigratePreprocessedData( val orgIds = (appConfigs.keys.filter { it.appId == DATA_COLLECTION_APP_ID }.map { it.organizationId } + LEGACY_ORG_ID).toSet() val principals = getChronicleSuperUserPrincipals() - val entitiesToWrite = orgIds.associateWith { getEntitiesForOrg(it, participants.getValue(it).associateBy { participant -> participant.participant_ek_id }, principals) }.values.flatten() + val invalidOrgIds = orgIds.filter { !participants.keys.contains(it) } + logger.info("Invalid organizations: ${invalidOrgIds.map { organizations[it] }}") + + val entitiesToWrite = (orgIds - invalidOrgIds.toSet()).associateWith { getEntitiesForOrg(it, participants.getValue(it).associateBy { participant -> participant.participant_ek_id }, principals) }.values.flatten() val written = writeEntities(entitiesToWrite) logger.info("Exported $written entities to preprocessed table") From 7c96721fd034c8c25092f1cc7f25310f9c739d17 Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Thu, 10 Mar 2022 04:02:41 -0800 Subject: [PATCH 91/99] invalid stuff --- .../openlattice/mechanic/upgrades/MigratePreprocessedData.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt index e625d588..67539a97 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt @@ -167,7 +167,7 @@ class MigratePreprocessedData( val orgIds = (appConfigs.keys.filter { it.appId == DATA_COLLECTION_APP_ID }.map { it.organizationId } + LEGACY_ORG_ID).toSet() val principals = getChronicleSuperUserPrincipals() val invalidOrgIds = orgIds.filter { !participants.keys.contains(it) } - logger.info("Invalid organizations: ${invalidOrgIds.map { organizations[it] }}") + logger.info("Organizations not found. Skipping: ${invalidOrgIds.map { organizations[it] }}") val entitiesToWrite = (orgIds - invalidOrgIds.toSet()).associateWith { getEntitiesForOrg(it, participants.getValue(it).associateBy { participant -> participant.participant_ek_id }, principals) }.values.flatten() From e9dd4027c56adfda5f0ac0d6171c1e14b0f90a1f Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Thu, 10 Mar 2022 04:04:27 -0800 Subject: [PATCH 92/99] log --- .../openlattice/mechanic/upgrades/MigratePreprocessedData.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt index 67539a97..4b858258 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt @@ -244,7 +244,7 @@ class MigratePreprocessedData( val allIds = participants.keys.toMutableSet() while (allIds.isNotEmpty()) { val current = allIds.take(50).toSet() - logger.info("processing neighbors of ${current.size} participants. Remaining ${(allIds - current).size}") + logger.info("processing batch of ${current.size}. Remaining: ${(allIds - current).size}") val participantNeighbors: Map> = getParticipantNeighbors( entityKeyIds = current, entitySetIds = orgEntitySetIds, From 5bafeb6b75ee428cc0a3b0370c09f4fba61b2c56 Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Thu, 10 Mar 2022 04:57:55 -0800 Subject: [PATCH 93/99] dont hold stuff in memory --- .../mechanic/upgrades/MigratePreprocessedData.kt | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt index 4b858258..5feec3ae 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt @@ -169,10 +169,8 @@ class MigratePreprocessedData( val invalidOrgIds = orgIds.filter { !participants.keys.contains(it) } logger.info("Organizations not found. Skipping: ${invalidOrgIds.map { organizations[it] }}") - val entitiesToWrite = (orgIds - invalidOrgIds.toSet()).associateWith { getEntitiesForOrg(it, participants.getValue(it).associateBy { participant -> participant.participant_ek_id }, principals) }.values.flatten() + val entitiesToWrite = (orgIds - invalidOrgIds.toSet()).forEach { exportEntities(it, participants.getValue(it).associateBy { participant -> participant.participant_ek_id }, principals) } - val written = writeEntities(entitiesToWrite) - logger.info("Exported $written entities to preprocessed table") return true } @@ -224,7 +222,7 @@ class MigratePreprocessedData( ) } - private fun getEntitiesForOrg(orgId: UUID, participants: Map, principals: Set): List { + private fun exportEntities(orgId: UUID, participants: Map, principals: Set) { logger.info("getting preprocessed data entities for org $orgId") val orgEntitySetIds = getOrgEntitySetNames(orgId) @@ -237,13 +235,12 @@ class MigratePreprocessedData( } if (participants.isEmpty()) { logger.info("No participants found. Skipping org") - return listOf() + return } - val result: MutableList = mutableListOf() val allIds = participants.keys.toMutableSet() while (allIds.isNotEmpty()) { - val current = allIds.take(50).toSet() + val current = allIds.take(20).toSet() logger.info("processing batch of ${current.size}. Remaining: ${(allIds - current).size}") val participantNeighbors: Map> = getParticipantNeighbors( entityKeyIds = current, @@ -256,12 +253,11 @@ class MigratePreprocessedData( }.values.flatten().filter { it.study_id != null || it.participant_id != null } logger.info("retrieved ${entities.size} preprocessed entities") - result.addAll(entities) + val written = writeEntities(entities) + logger.info("exported $written entities to table") allIds -= current } - logger.info("Total preprocessed entities retrieved for org: ${result.size}") - return result } private fun getEntity( From d81dc8a2743e424cf39b0e9ebf7e062318b3795a Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Thu, 10 Mar 2022 05:09:27 -0800 Subject: [PATCH 94/99] add batch --- .../com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt index 5feec3ae..0c0ef356 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt @@ -196,6 +196,7 @@ class MigratePreprocessedData( ps.setBoolean(++index, it.newApp) ps.setObject(++index, it.duration) ps.setString(++index, it.warning) + ps.addBatch() } ps.executeBatch().sum() } From 43a15a6b73df3ebc7f1fe50672c02d8af26bb797 Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Thu, 10 Mar 2022 05:52:29 -0800 Subject: [PATCH 95/99] close connection --- .../openlattice/mechanic/upgrades/MigratePreprocessedData.kt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt index 0c0ef356..26210637 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt @@ -179,7 +179,8 @@ class MigratePreprocessedData( } private fun writeEntities(entities: List): Int { - return getHikariDataSource().connection.use { connection -> + val hds = getHikariDataSource() + return hds.connection.use { connection -> try { val wc = connection.prepareStatement(INSERT_SQL).use { ps -> entities.forEach { @@ -200,6 +201,7 @@ class MigratePreprocessedData( } ps.executeBatch().sum() } + hds.connection.close() return@use wc } catch (ex: Exception) { logger.error("exception", ex) From b7a66a65e19b5c289d44457ea2e38438d9844eea Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Thu, 10 Mar 2022 09:38:23 -0800 Subject: [PATCH 96/99] pass hds --- .../upgrades/MigratePreprocessedData.kt | 27 +++++++++++++------ 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt index 26210637..d3ff5abe 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt @@ -169,7 +169,15 @@ class MigratePreprocessedData( val invalidOrgIds = orgIds.filter { !participants.keys.contains(it) } logger.info("Organizations not found. Skipping: ${invalidOrgIds.map { organizations[it] }}") - val entitiesToWrite = (orgIds - invalidOrgIds.toSet()).forEach { exportEntities(it, participants.getValue(it).associateBy { participant -> participant.participant_ek_id }, principals) } + val hds = getHikariDataSource() + (orgIds - invalidOrgIds.toSet()).forEach { orgId -> + exportEntities( + hds, + orgId, + participants.getValue(orgId).associateBy { participant -> participant.participant_ek_id }, + principals + ) + } return true } @@ -178,8 +186,7 @@ class MigratePreprocessedData( return Version.V2021_07_23.value } - private fun writeEntities(entities: List): Int { - val hds = getHikariDataSource() + private fun writeEntities(entities: List, hds: HikariDataSource): Int { return hds.connection.use { connection -> try { val wc = connection.prepareStatement(INSERT_SQL).use { ps -> @@ -201,7 +208,6 @@ class MigratePreprocessedData( } ps.executeBatch().sum() } - hds.connection.close() return@use wc } catch (ex: Exception) { logger.error("exception", ex) @@ -225,14 +231,19 @@ class MigratePreprocessedData( ) } - private fun exportEntities(orgId: UUID, participants: Map, principals: Set) { + private fun exportEntities( + hds: HikariDataSource, + orgId: UUID, + participants: Map, + principals: Set + ) { logger.info("getting preprocessed data entities for org $orgId") val orgEntitySetIds = getOrgEntitySetNames(orgId) val participantEntitySetIds = when (orgId) { LEGACY_ORG_ID -> { val entitySetNames = participants.values.map { it.legacy_study_id }.map { studyId -> "chronicle_participants_$studyId" } - entitySetIds.filter { entitySetNames.contains(it.key) }.values.toSet() + entitySetIds.filter { entitySetNames.contains(it.key) }.values.toSet() } else -> setOf(orgEntitySetIds.getValue(PARTICIPANTS_ES)) } @@ -251,12 +262,12 @@ class MigratePreprocessedData( participantEntitySetIds = participantEntitySetIds, principals = principals ) - val entities = participantNeighbors.mapValues { + val entities = participantNeighbors.mapValues { it.value.map { entityDetails -> getEntity(entityDetails.neighborDetails.get(), it.key, participants) } }.values.flatten().filter { it.study_id != null || it.participant_id != null } logger.info("retrieved ${entities.size} preprocessed entities") - val written = writeEntities(entities) + val written = writeEntities(entities, hds) logger.info("exported $written entities to table") allIds -= current From dffe874e2e750f6731f652235db142ea10f83646 Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Fri, 11 Mar 2022 18:49:34 -0800 Subject: [PATCH 97/99] select all participants --- .../openlattice/mechanic/upgrades/MigratePreprocessedData.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt index d3ff5abe..e4a09a5b 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt @@ -158,7 +158,7 @@ class MigratePreprocessedData( val participants = BasePostgresIterable( PreparedStatementHolderSupplier( getHikariDataSource(), - "SELECT * FROM participants_export" + "select * from (select * from participants_export union select * from missed_participants_fresh) AS participants" ) {} ) { participant(it) From ccc0f97effc3c6f84e298cb5d04300d7b3679aeb Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Sat, 12 Mar 2022 05:17:27 -0800 Subject: [PATCH 98/99] filter out null stuff --- .../openlattice/mechanic/upgrades/MigratePreprocessedData.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt index e4a09a5b..859f9683 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt @@ -264,7 +264,7 @@ class MigratePreprocessedData( ) val entities = participantNeighbors.mapValues { it.value.map { entityDetails -> getEntity(entityDetails.neighborDetails.get(), it.key, participants) } - }.values.flatten().filter { it.study_id != null || it.participant_id != null } + }.values.flatten().filter { it.study_id != null && it.participant_id != null && it.packageName != null && it.packageName.isNotBlank() } logger.info("retrieved ${entities.size} preprocessed entities") val written = writeEntities(entities, hds) From c282d7c13ec81a4072ba44c86a40544e05befac9 Mon Sep 17 00:00:00 2001 From: Alfonce Nzioka Date: Sat, 12 Mar 2022 05:30:32 -0800 Subject: [PATCH 99/99] fix --- .../openlattice/mechanic/upgrades/MigratePreprocessedData.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt index 859f9683..f2e1b443 100644 --- a/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt +++ b/src/main/kotlin/com/openlattice/mechanic/upgrades/MigratePreprocessedData.kt @@ -264,7 +264,7 @@ class MigratePreprocessedData( ) val entities = participantNeighbors.mapValues { it.value.map { entityDetails -> getEntity(entityDetails.neighborDetails.get(), it.key, participants) } - }.values.flatten().filter { it.study_id != null && it.participant_id != null && it.packageName != null && it.packageName.isNotBlank() } + }.values.flatten().filter { it.study_id != null && it.participant_id != null} logger.info("retrieved ${entities.size} preprocessed entities") val written = writeEntities(entities, hds)