Skip to content

PIR: Add support for multiple profile queries in PirScan and PirOptOut #6413

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Jul 21, 2025
Merged

PIR: Add support for multiple profile queries in PirScan and PirOptOut #6413

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
207758b
Add support for multiple profile queries in PirScan and PirOptOut
landomen Jul 17, 2025
575be3a
Add additional default profile to PirOptOut
landomen Jul 17, 2025
7daa1ef
Fix code formatting
landomen Jul 17, 2025
8e40765
Optimize the runner work distribution
landomen Jul 21, 2025
a8a7496
Update debug opt-out
landomen Jul 21, 2025
2133953
Simplify parse steps function to use profile id
landomen Jul 21, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 15 additions & 6 deletions pir/pir-internal/src/main/java/com/duckduckgo/pir/internal/common/BrokerStepsParser.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import com.duckduckgo.pir.internal.common.BrokerStepsParser.BrokerStep.OptOutSte
import com.duckduckgo.pir.internal.common.BrokerStepsParser.BrokerStep.ScanStep
import com.duckduckgo.pir.internal.scripts.models.BrokerAction
import com.duckduckgo.pir.internal.scripts.models.ExtractedProfile
import com.duckduckgo.pir.internal.scripts.models.ProfileQuery
import com.duckduckgo.pir.internal.store.PirRepository
import com.squareup.anvil.annotations.ContributesBinding
import com.squareup.moshi.JsonAdapter
Expand All @@ -40,12 +41,14 @@ interface BrokerStepsParser {
*
* @param brokerName - name of the broker to which these steps belong to
* @param stepsJson - string in JSONObject format obtained from the broker's json representing a step (scan / opt-out).
* @param profileQuery - profile associated with the step (used for the opt-out step)
* @return list of broker steps resulting from the passed params. If the step is of type OptOut, it will return a list of
* OptOutSteps where an OptOut step is mapped to each of the profile for the broker.
*/
suspend fun parseStep(
brokerName: String,
stepsJson: String,
profileQuery: ProfileQuery? = null,
): List<BrokerStep>

sealed class BrokerStep(
Expand Down Expand Up @@ -101,16 +104,22 @@ class RealBrokerStepsParser @Inject constructor(
override suspend fun parseStep(
brokerName: String,
stepsJson: String,
profileQuery: ProfileQuery?,
): List<BrokerStep> = withContext(dispatcherProvider.io()) {
return@withContext runCatching {
adapter.fromJson(stepsJson)?.run {
if (this is OptOutStep) {
repository.getExtractProfileResultForBroker(brokerName)?.extractResults?.map {
this.copy(
brokerName = brokerName,
profileToOptOut = it,
)
}
repository.getExtractProfileResultsForBroker(brokerName)
.filter { it.profileQuery != null && it.profileQuery.id == profileQuery?.id }
.map {
it.extractResults.map { extractedProfile ->
this.copy(
brokerName = brokerName,
profileToOptOut = extractedProfile,
)
}
}
.flatten()
} else {
listOf((this as ScanStep).copy(brokerName = brokerName))
}
Expand Down
8 changes: 6 additions & 2 deletions pir/pir-internal/src/main/java/com/duckduckgo/pir/internal/common/PirActionsRunner.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -145,8 +145,12 @@ class RealPirActionsRunner @AssistedInject constructor(
}

withContext(dispatcherProvider.main()) {
logcat { "PIR-RUNNER (${this@RealPirActionsRunner}): ${Thread.currentThread().name} Brokers to execute $brokerSteps" }
logcat { "PIR-RUNNER (${this@RealPirActionsRunner}): ${Thread.currentThread().name} Brokers size: ${brokerSteps.size}" }
logcat {
"PIR-RUNNER (${this@RealPirActionsRunner}): ${Thread.currentThread().name} " +
"Brokers size: ${brokerSteps.size} " +
"profile=$profileQuery " +
"Brokers to execute $brokerSteps"
}
detachedWebView = pirDetachedWebViewProvider.createInstance(
context,
pirScriptToLoad,
Expand Down
198 changes: 124 additions & 74 deletions pir/pir-internal/src/main/java/com/duckduckgo/pir/internal/optout/PirOptOut.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,21 +101,58 @@ class RealPirOptOut @Inject constructor(
private val dispatcherProvider: DispatcherProvider,
callbacks: PluginPoint<PirCallbacks>,
) : PirOptOut, PirJob(callbacks) {
private var profileQuery: ProfileQuery = ProfileQuery(
firstName = "William",
lastName = "Smith",
city = "Chicago",
state = "IL",
addresses = listOf(
Address(
city = "Chicago",
state = "IL",
private var profileQueries: List<ProfileQuery> = listOf(
ProfileQuery(
id = -1,
firstName = "William",
lastName = "Smith",
city = "Chicago",
state = "IL",
addresses = listOf(
Address(
city = "Chicago",
state = "IL",
),
),
birthYear = 1993,
fullName = "William Smith",
age = 32,
deprecated = false,
),
ProfileQuery(
id = -2,
firstName = "Jane",
lastName = "Doe",
city = "New York",
state = "NY",
addresses = listOf(
Address(
city = "New York",
state = "NY",
),
),
birthYear = 1990,
fullName = "Jane Doe",
age = 35,
deprecated = false,
),
ProfileQuery(
id = -3,
firstName = "Alicia",
lastName = "West",
city = "Los Angeles",
state = "CA",
addresses = listOf(
Address(
city = "Los Angeles",
state = "CA",
),
),
birthYear = 1985,
fullName = "Alicia West",
age = 40,
deprecated = false,
),
birthYear = 1993,
fullName = "William Smith",
age = 32,
deprecated = false,
)

private val runners: MutableList<PirActionsRunner> = mutableListOf()
Expand All @@ -131,9 +168,9 @@ class RealPirOptOut @Inject constructor(
cleanRunners()
runners.clear()
}
obtainProfile()
obtainProfiles()

logcat { "PIR-OPT-OUT: Running opt-out on profile: $profileQuery on ${Thread.currentThread().name}" }
logcat { "PIR-OPT-OUT: Running debug opt-out for $brokers on profiles: $profileQueries on ${Thread.currentThread().name}" }

runners.add(
pirActionsRunnerFactory.create(
Expand All @@ -143,21 +180,28 @@ class RealPirOptOut @Inject constructor(
),
)

// Start each runner on a subset of the broker steps
// Load opt-out steps jsons for each broker
val brokerOptOutStepsJsons = brokers.mapNotNull { broker ->
repository.getBrokerOptOutSteps(broker)?.let { broker to it }
}

brokers.mapNotNull { broker ->
repository.getBrokerOptOutSteps(broker)?.run {
brokerStepsParser.parseStep(broker, this)
}
}.filter {
it.isNotEmpty()
}.flatten()
.also { list ->
runners[0].startOn(webView, profileQuery, list)
runners[0].stop()
}
// Create a map of profiles that have opt-out steps for those brokers
val profileBrokerStepsMap = profileQueries.associateWith { profileQuery ->
brokerOptOutStepsJsons.map { (broker, stepsJson) ->
brokerStepsParser.parseStep(broker, stepsJson, profileQuery)
}.flatten()
}.filter { it.value.isNotEmpty() }

// Start each runner on the profile with the broker steps
profileBrokerStepsMap.entries.map { (profileQuery, brokerSteps) ->
logcat { "PIR-OPT-OUT: Start opt-out for profile: $profileQuery on ${Thread.currentThread().name}" }
runners[0].startOn(webView, profileQuery, brokerSteps)
runners[0].stop()
logcat { "PIR-OPT-OUT: Finish opt-out for profile: $profileQuery on ${Thread.currentThread().name}" }
}

logcat { "PIR-OPT-OUT: Opt-out completed for all runners and profiles" }

logcat { "PIR-OPT-OUT: Optout completed for all runners" }
emitCompletedPixel()
onJobCompleted()
return@withContext Result.success(Unit)
Expand All @@ -173,26 +217,27 @@ class RealPirOptOut @Inject constructor(
cleanRunners()
runners.clear()
}
obtainProfile()
obtainProfiles()

logcat { "PIR-OPT-OUT: Running opt-out on profile: $profileQuery on ${Thread.currentThread().name}" }
logcat { "PIR-OPT-OUT: Running opt-out on profiles: $profileQueries on ${Thread.currentThread().name}" }

val script = pirCssScriptLoader.getScript()

val brokerSteps = brokers.mapNotNull { broker ->
repository.getBrokerOptOutSteps(broker)?.run {
brokerStepsParser.parseStep(broker, this)
}
}.filter {
it.isNotEmpty()
}.flatten()

maxWebViewCount = if (brokerSteps.size <= MAX_DETACHED_WEBVIEW_COUNT) {
brokerSteps.size
} else {
MAX_DETACHED_WEBVIEW_COUNT
// Load opt-out steps jsons for each broker
val brokerOptOutStepsJsons = brokers.mapNotNull { broker ->
repository.getBrokerOptOutSteps(broker)?.let { broker to it }
}

// Create a map of profiles that have opt-out steps for those brokers
val profileBrokerStepsMap = profileQueries.associateWith { profileQuery ->
brokerOptOutStepsJsons.map { (broker, stepsJson) ->
brokerStepsParser.parseStep(broker, stepsJson, profileQuery)
}.flatten()
}.filter { it.value.isNotEmpty() }

val stepsCount = profileBrokerStepsMap.values.maxOf { it.size }
maxWebViewCount = minOf(stepsCount, MAX_DETACHED_WEBVIEW_COUNT)

logcat { "PIR-OPT-OUT: Attempting to create $maxWebViewCount parallel runners on ${Thread.currentThread().name}" }

// Initiate runners
Expand All @@ -208,45 +253,50 @@ class RealPirOptOut @Inject constructor(
createCount++
}

// Start each runner on a subset of the broker steps
brokerSteps.splitIntoParts(maxWebViewCount)
.mapIndexed { index, part ->
async {
runners[index].start(profileQuery, part)
runners[index].stop()
}
}.awaitAll()
// Start each runner on the profile with the broker steps
profileBrokerStepsMap.entries.map { (profileQuery, brokerSteps) ->
logcat { "PIR-OPT-OUT: Start opt-out on profile: $profileQuery" }
brokerSteps.splitIntoParts(maxWebViewCount)
.mapIndexed { index, part ->
async {
runners[index].start(profileQuery, part)
runners[index].stop()
}
}.awaitAll()
logcat { "PIR-OPT-OUT: Finish opt-out on profile: $profileQuery" }
}

logcat { "PIR-OPT-OUT: Optout completed for all runners" }
logcat { "PIR-OPT-OUT: Opt-out completed for all runners and profiles" }
emitCompletedPixel()
onJobCompleted()
return@withContext Result.success(Unit)
}

private suspend fun obtainProfile() {
repository.getUserProfiles().also {
if (it.isNotEmpty()) {
// Temporarily taking the first profile only for the PoC. In the reality, more than 1 should be allowed.
val storedProfile = it[0]
profileQuery = ProfileQuery(
firstName = storedProfile.userName.firstName,
lastName = storedProfile.userName.lastName,
city = storedProfile.addresses.city,
state = storedProfile.addresses.state,
addresses = listOf(
Address(
city = storedProfile.addresses.city,
state = storedProfile.addresses.state,
private suspend fun obtainProfiles() {
repository.getUserProfiles().also { profiles ->
if (profiles.isNotEmpty()) {
profileQueries = profiles.map { storedProfile ->
ProfileQuery(
id = storedProfile.id,
firstName = storedProfile.userName.firstName,
lastName = storedProfile.userName.lastName,
city = storedProfile.addresses.city,
state = storedProfile.addresses.state,
addresses = listOf(
Address(
city = storedProfile.addresses.city,
state = storedProfile.addresses.state,
),
),
),
birthYear = storedProfile.birthYear,
fullName = storedProfile.userName.middleName?.run {
"${storedProfile.userName.firstName} $this ${storedProfile.userName.lastName}"
}
?: "${storedProfile.userName.firstName} ${storedProfile.userName.lastName}",
age = LocalDate.now().year - storedProfile.birthYear,
deprecated = false,
)
birthYear = storedProfile.birthYear,
fullName = storedProfile.userName.middleName?.run {
"${storedProfile.userName.firstName} $this ${storedProfile.userName.lastName}"
}
?: "${storedProfile.userName.firstName} ${storedProfile.userName.lastName}",
age = LocalDate.now().year - storedProfile.birthYear,
deprecated = false,
)
}
}
}
}
Expand Down
Loading
Loading