Skip to content

fix(browser): extract recipes from pages behind a bot check #521

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
plusmobileapps wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix(browser): extract recipes from pages behind a bot check #521

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions ...er/impl/src/commonMain/kotlin/com/plusmobileapps/chefmate/browser/impl/BrowserBlocImpl.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ class BrowserBlocImpl(
canGoForward = it.canGoForward,
goBackTrigger = it.goBackTrigger,
goForwardTrigger = it.goForwardTrigger,
captureHtmlTrigger = it.captureHtmlTrigger,
)
}

Expand All @@ -64,6 +65,10 @@ class BrowserBlocImpl(
viewModel.extractRecipe()
}

override fun onHtmlCaptured(html: String?) {
viewModel.onHtmlCaptured(html)
}

override fun onDismissMessage() {
viewModel.dismissMessage()
}
Expand Down
49 changes: 47 additions & 2 deletions ...r/impl/src/commonMain/kotlin/com/plusmobileapps/chefmate/browser/impl/BrowserViewModel.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,13 @@ import com.plusmobileapps.chefmate.browser.BrowserHistoryRepository
import com.plusmobileapps.chefmate.di.Main
import dev.zacsweers.metro.Inject
import kotlin.coroutines.CoroutineContext
import kotlinx.coroutines.CompletableDeferred
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.StateFlow
import kotlinx.coroutines.flow.asStateFlow
import kotlinx.coroutines.launch
import kotlinx.coroutines.withTimeoutOrNull
import kotlinx.serialization.json.Json

@Inject
class BrowserViewModel(
Expand All @@ -25,6 +28,8 @@ class BrowserViewModel(

private var output: Consumer<BrowserBloc.Output>? = null

private var pendingHtmlCapture: CompletableDeferred<String?>? = null

fun setOutput(output: Consumer<BrowserBloc.Output>) {
this.output = output
}
Expand Down Expand Up @@ -54,11 +59,24 @@ class BrowserViewModel(
val url = _state.value.webViewReportedUrl.ifBlank { _state.value.currentUrl }
if (url.isBlank() || _state.value.isExtracting) return

_state.value = _state.value.copy(isExtracting = true, extractionFailed = false)
val capture = CompletableDeferred<String?>()
pendingHtmlCapture = capture
_state.value =
_state.value.copy(
isExtracting = true,
extractionFailed = false,
captureHtmlTrigger = _state.value.captureHtmlTrigger + 1,
)

scope.launch {
// The WebView holds the only copy of the page as the user actually sees it — past any
// bot-check interstitial, and with client-rendered markup in place — so hand that to
// the extractor. A WebView that never answers must not strand the user on a spinner,
// so give up waiting and let the extractor fall back to fetching the URL itself.
val renderedHtml = withTimeoutOrNull(HTML_CAPTURE_TIMEOUT_MS) { capture.await() }
pendingHtmlCapture = null
try {
val extracted = recipeExtractorService.extractRecipe(url)
val extracted = recipeExtractorService.extractRecipe(url, renderedHtml)
_state.value = _state.value.copy(isExtracting = false)
output?.onNext(BrowserBloc.Output.RecipeExtracted(extracted))
} catch (e: Exception) {
Expand All @@ -68,6 +86,11 @@ class BrowserViewModel(
}
}

/** Called from the WebView for each `captureHtmlTrigger`, on an arbitrary thread. */
fun onHtmlCaptured(html: String?) {
pendingHtmlCapture?.complete(decodeCapturedHtml(html))
}

fun dismissMessage() {
_state.value = _state.value.copy(extractionFailed = false)
}
Expand Down Expand Up @@ -107,9 +130,31 @@ class BrowserViewModel(
val canGoForward: Boolean = false,
val goBackTrigger: Int = 0,
val goForwardTrigger: Int = 0,
val captureHtmlTrigger: Int = 0,
)

companion object {
private const val DEFAULT_URL = "https://www.google.com"

/** Generous enough for a large page to serialise, short enough not to read as a hang. */
private const val HTML_CAPTURE_TIMEOUT_MS = 5_000L

/**
* Android hands a script's result back JSON-encoded (`"<html>…"`); iOS and the JavaFX
* engine hand back the string itself. Normalises both, and treats JavaScript's `null` —
* what Android sends when the script failed or its result was too large to marshal — as
* nothing captured.
*/
internal fun decodeCapturedHtml(raw: String?): String? {
val trimmed = raw?.trim().orEmpty()
if (trimmed.isEmpty() || trimmed == "null") return null
val decoded =
if (trimmed.startsWith("\"") && trimmed.endsWith("\"")) {
runCatching { Json.decodeFromString<String>(trimmed) }.getOrNull()
} else {
trimmed
}
return decoded?.takeIf { it.isNotBlank() }
}
}
}
186 changes: 116 additions & 70 deletions .../src/commonMain/kotlin/com/plusmobileapps/chefmate/browser/impl/RecipeExtractorService.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,13 +27,16 @@ import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonArray
import kotlinx.serialization.json.JsonElement
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.JsonPrimitive
import kotlinx.serialization.json.contentOrNull
import kotlinx.serialization.json.jsonArray
import kotlinx.serialization.json.jsonObject
import kotlinx.serialization.json.jsonPrimitive

interface RecipeExtractorService {
suspend fun extractRecipe(url: String): ExtractedRecipeData
/**
* @param renderedHtml The page's markup as the in-app WebView rendered it, or `null` when it
* couldn't be read. Preferred over fetching [url], which for a growing number of sites
* returns a bot-check page rather than the recipe.
*/
suspend fun extractRecipe(url: String, renderedHtml: String?): ExtractedRecipeData
}

@SingleIn(AppScope::class)
Expand All @@ -48,45 +51,58 @@ class RecipeExtractorServiceImpl(

private val json = Json { ignoreUnknownKeys = true }

override suspend fun extractRecipe(url: String): ExtractedRecipeData =
override suspend fun extractRecipe(url: String, renderedHtml: String?): ExtractedRecipeData =
withContext(ioContext) {
// Our own share links render their recipe content client-side after load (fetched from
// Supabase), so there's no server-rendered markup for the scraper below to parse —
// fetch the recipe directly instead of scraping the page.
ChefMateUrls.recipeShareUrlRemoteId(url)?.let { remoteId ->
return@withContext recipeRepository
.fetchPublicRecipe(remoteId)
.getOrElse { throw IllegalStateException("No recipe data found on this page") }
.getOrElse { throw IllegalStateException(NO_RECIPE_FOUND) }
.toExtractedRecipeData(url)
}

val response = httpClient.get(url) { header("User-Agent", USER_AGENT) }
if (!response.status.isSuccess()) {
throw IllegalStateException(
"Could not load the page (HTTP ${response.status.value})"
)
}
val html = response.bodyAsText()

val document = Ksoup.parse(html)
val jsonLdScripts = document.select("script[type=application/ld+json]")
// The rendered DOM is strictly better source material than a fresh fetch — it's the
// page after its scripts ran, and after whatever bot check stood in front of it.
renderedHtml
?.let { parseRecipe(Ksoup.parse(it), url) }
?.let {
return@withContext it
}

for (script in jsonLdScripts) {
val data = parseRecipeJsonText(script.data(), url) ?: continue
// schema.org flattens grouped ingredients, so recover any sub-section headers
// (e.g. "For the sauce:") from the page markup when present.
val grouped = parseIngredientSections(document)
return@withContext if (grouped != null) data.copy(ingredients = grouped) else data
val fetched = runCatching {
val response = httpClient.get(url) { header("User-Agent", USER_AGENT) }
if (!response.status.isSuccess()) {
throw IllegalStateException(
"Could not load the page (HTTP ${response.status.value})"
)
}
response.bodyAsText()
}

// No JSON-LD recipe — fall back to microdata/microformat markup.
parseMicrodataRecipe(document, url)?.let {
return@withContext it
}
// Having already read the rendered page, a failed fetch tells the user nothing useful —
// the page they're looking at simply had no recipe markup in it.
val html = if (renderedHtml != null) fetched.getOrNull() else fetched.getOrThrow()

throw IllegalStateException("No recipe data found on this page")
html?.let { parseRecipe(Ksoup.parse(it), url) }
?: throw IllegalStateException(NO_RECIPE_FOUND)
}

/** Returns `null` when [document] carries no recipe markup this parser understands. */
private fun parseRecipe(document: Document, url: String): ExtractedRecipeData? {
for (script in document.select("script[type=application/ld+json]")) {
val data = parseRecipeJsonText(script.data(), url) ?: continue
// schema.org flattens grouped ingredients, so recover any sub-section headers
// (e.g. "For the sauce:") from the page markup when present.
val grouped = parseIngredientSections(document)
return if (grouped != null) data.copy(ingredients = grouped) else data
}

// No JSON-LD recipe — fall back to microdata/microformat markup.
return parseMicrodataRecipe(document, url)
}

private fun Recipe.toExtractedRecipeData(sourceUrl: String): ExtractedRecipeData =
ExtractedRecipeData(
title = title,
Expand Down Expand Up @@ -258,10 +274,10 @@ class RecipeExtractorServiceImpl(
is JsonObject -> {
val type = element["@type"]
val isRecipe =
when {
type == null -> false
type is JsonArray -> type.any { it.jsonPrimitive.contentOrNull == "Recipe" }
else -> type.jsonPrimitive.contentOrNull == "Recipe"
when (type) {
null -> false
is JsonArray -> type.any { it.stringOrNull() == "Recipe" }
else -> type.stringOrNull() == "Recipe"
}
if (isRecipe) {
element
Expand Down Expand Up @@ -294,27 +310,30 @@ class RecipeExtractorServiceImpl(
private fun String.decodeHtmlEntities(): String = Parser.unescapeEntities(this, false)

private fun parseRecipeFromJson(obj: JsonObject, sourceUrl: String): ExtractedRecipeData {
val title = obj["name"]?.jsonPrimitive?.contentOrNull?.decodeHtmlEntities() ?: ""
val description = obj["description"]?.jsonPrimitive?.contentOrNull?.decodeHtmlEntities()
// `headline` is the fallback for publishers that only title the article, not the recipe.
val title =
(obj["name"].stringOrNull() ?: obj["headline"].stringOrNull())?.decodeHtmlEntities()
?: ""
val description = obj["description"].stringOrNull()?.decodeHtmlEntities()

val ingredients =
obj["recipeIngredient"]?.jsonArray?.mapNotNull {
it.jsonPrimitive.contentOrNull?.decodeHtmlEntities()
} ?: emptyList()
(obj["recipeIngredient"] ?: obj["ingredients"]).stringList().map {
it.decodeHtmlEntities()
}

val directions = parseDirections(obj)

val imageUrl =
when (val img = obj["image"]) {
is JsonArray -> img.firstOrNull()?.let { extractImageUrl(it) }
is JsonObject -> extractImageUrl(img)
else -> img?.jsonPrimitive?.contentOrNull
else -> img.stringOrNull()
}

val servings = parseServings(obj)
val prepTime = parseDuration(obj["prepTime"]?.jsonPrimitive?.contentOrNull)
val cookTime = parseDuration(obj["cookTime"]?.jsonPrimitive?.contentOrNull)
val totalTime = parseDuration(obj["totalTime"]?.jsonPrimitive?.contentOrNull)
val prepTime = obj["prepTime"].durationMinutes()
val cookTime = obj["cookTime"].durationMinutes()
val totalTime = obj["totalTime"].durationMinutes()
val calories = parseCalories(obj)

return ExtractedRecipeData(
Expand All @@ -338,52 +357,46 @@ class RecipeExtractorServiceImpl(
is JsonArray ->
instructions.flatMap { element ->
when {
element is JsonObject &&
element["@type"]?.jsonPrimitive?.contentOrNull == "HowToStep" ->
element !is JsonObject ->
listOfNotNull(element.stringOrNull()?.decodeHtmlEntities())
element["@type"].stringOrNull() == "HowToSection" ->
(element["itemListElement"] as? JsonArray)?.mapNotNull { item ->
(item as? JsonObject)?.get("text").stringOrNull()
} ?: emptyList()
// Anything else object-shaped is a step: HowToStep, CreativeWork, or an
// untyped `{"text": …}`. Read `text`, falling back to `name` for the
// publishers that put the step body there instead.
else ->
listOfNotNull(
element["text"]?.jsonPrimitive?.contentOrNull?.decodeHtmlEntities()
element["text"].stringOrNull() ?: element["name"].stringOrNull()
)
element is JsonObject &&
element["@type"]?.jsonPrimitive?.contentOrNull == "HowToSection" -> {
val items =
element["itemListElement"]?.jsonArray ?: return@flatMap emptyList()
items.mapNotNull { item ->
item.jsonObject["text"]
?.jsonPrimitive
?.contentOrNull
?.decodeHtmlEntities()
}
}
else ->
listOfNotNull(element.jsonPrimitive.contentOrNull?.decodeHtmlEntities())
}
}.map { it.decodeHtmlEntities() }
}
else -> listOfNotNull(instructions.jsonPrimitive.contentOrNull?.decodeHtmlEntities())
else -> listOfNotNull(instructions.stringOrNull()?.decodeHtmlEntities())
}
}

private fun extractImageUrl(element: JsonElement): String? =
when (element) {
is JsonObject -> element["url"]?.jsonPrimitive?.contentOrNull
else -> element.jsonPrimitive.contentOrNull
is JsonObject -> element["url"].stringOrNull()
else -> element.stringOrNull()
}

private fun parseServings(obj: JsonObject): Int? {
val yield = obj["recipeYield"] ?: return null
return when (yield) {
is JsonArray -> yield.firstOrNull()?.jsonPrimitive?.contentOrNull.firstNumber()
else -> yield.jsonPrimitive.contentOrNull.firstNumber()
private fun parseServings(obj: JsonObject): Int? =
when (val yield = obj["recipeYield"]) {
null -> null
is JsonArray -> yield.firstOrNull().stringOrNull().firstNumber()
else -> yield.stringOrNull().firstNumber()
}
}

private fun parseCalories(obj: JsonObject): Int? {
val nutrition = obj["nutrition"]?.jsonObject ?: return null
return nutrition["calories"]?.jsonPrimitive?.contentOrNull.firstNumber()
}
private fun parseCalories(obj: JsonObject): Int? =
(obj["nutrition"] as? JsonObject)?.get("calories").stringOrNull().firstNumber()

companion object {
// A real mobile-browser User-Agent. Some sites' WAFs return 403 to bot-style or
// spoofed desktop-Chrome User-Agents, but accept genuine mobile-browser strings.
private const val NO_RECIPE_FOUND = "No recipe data found on this page"

private const val USER_AGENT =
"Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 " +
"(KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1"
Expand Down Expand Up @@ -420,6 +433,39 @@ class RecipeExtractorServiceImpl(

private fun String?.nullIfBlank(): String? = this?.takeIf { it.isNotBlank() }

/**
* schema.org values are only *conventionally* strings — publishers routinely nest an object
* or an array where the spec suggests text. Reading them through this (rather than
* `jsonPrimitive`, which throws) keeps one oddly-shaped field from failing the whole
* recipe.
*/
private fun JsonElement?.stringOrNull(): String? = (this as? JsonPrimitive)?.contentOrNull

/** Reads a field that may be published as a single string or a list of them. */
private fun JsonElement?.stringList(): List<String> =
when (this) {
null -> emptyList()
is JsonArray -> mapNotNull { it.stringOrNull() }
else -> listOfNotNull(stringOrNull())
}

/**
* A schema.org duration is usually a bare ISO-8601 string, but a recipe that publishes a
* *range* — Serious Eats' sous vide burgers cook for anywhere from 45 minutes to 4 hours —
* uses a `Duration` object with `minValue`/`maxValue` instead. Take the lower bound, the
* same end of a range [firstNumber] takes for a "6 to 8" yield.
*/
private fun JsonElement?.durationMinutes(): Int? =
when (this) {
null -> null
is JsonArray -> firstOrNull().durationMinutes()
is JsonObject ->
parseDuration(this["minValue"].stringOrNull())
?: parseDuration(this["value"].stringOrNull())
?: parseDuration(this["maxValue"].stringOrNull())
else -> parseDuration(stringOrNull())
}

private fun String.collapseWhitespace(): String = replace(WHITESPACE, " ").trim()

/**
Expand Down
Loading
Loading