Skip to content

Technologies sort fix for the category overview #120

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 6 commits into
base: main
Choose a base branch
from
Open

Technologies sort fix for the category overview #120

Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
cf1fae5
report adjusted
max-ostapenko Jun 30, 2025
7173b0a
cleanup
max-ostapenko Jun 30, 2025
7683f23
flattenned results
max-ostapenko Jun 30, 2025
d8efee5
Merge branch 'main' into wasteful-macaw
max-ostapenko Jul 6, 2025
84e0d5f
Merge branch 'main' into wasteful-macaw
max-ostapenko Jul 6, 2025
4fb85fb
Merge branch 'main' into wasteful-macaw
max-ostapenko Jul 8, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
75 changes: 31 additions & 44 deletions definitions/output/reports/tech_report_categories.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,11 @@ publish('tech_report_categories', {
WITH pages AS (
SELECT DISTINCT
client,
root_page,
technologies
FROM ${ctx.ref('crawl', 'pages')}
category,
root_page
FROM ${ctx.ref('crawl', 'pages')} AS pages
INNER JOIN pages.technologies AS tech
INNER JOIN tech.categories AS category
WHERE
date = '${pastMonth}'
${constants.devRankFilter}
Expand All @@ -36,7 +38,7 @@ crux AS (
merged_pages AS (
SELECT DISTINCT
client,
technologies,
category,
root_page
FROM pages
INNER JOIN crux
Expand All @@ -45,68 +47,53 @@ merged_pages AS (

category_stats AS (
SELECT
client,
category,
STRUCT(
MAX(IF(client = 'desktop', origins, 0)) AS desktop,
MAX(IF(client = 'mobile', origins, 0)) AS mobile
) AS origins
FROM (
SELECT
client,
category,
COUNT(DISTINCT root_page) AS origins
FROM merged_pages
INNER JOIN merged_pages.technologies AS tech
INNER JOIN tech.categories AS category
WHERE
category IS NOT NULL
GROUP BY
client,
category
)
GROUP BY category
COUNT(DISTINCT root_page) AS origins
FROM merged_pages
GROUP BY
client,
category
),

technology_stats AS (
SELECT
technology,
category_obj AS categories,
origins.mobile AS mobile_origins
FROM ${ctx.ref('reports', 'tech_report_technologies')}
category,
STRUCT(
ARRAY_AGG(technology IGNORE NULLS ORDER BY origins.mobile DESC) AS mobile,
ARRAY_AGG(technology IGNORE NULLS ORDER BY origins.desktop DESC) AS desktop
) AS technologies
FROM ${ctx.ref('reports', 'tech_report_technologies')},
UNNEST(category_obj) AS category
GROUP BY category
)

SELECT
client,
category,
description,
origins,
ARRAY_AGG(technology IGNORE NULLS ORDER BY technology_stats.mobile_origins DESC) AS technologies
IF(
client = 'mobile',
technologies.mobile,
technologies.desktop
) AS technologies
FROM category_stats
INNER JOIN technology_stats
ON category_stats.category IN UNNEST(technology_stats.categories)
USING (category)
INNER JOIN category_descriptions
USING (category)
GROUP BY
category,
description,
origins

UNION ALL

SELECT
client,
'ALL' AS category,
NULL AS description,
STRUCT(
MAX(IF(client = 'desktop', origins, 0)) AS desktop,
MAX(IF(client = 'mobile', origins, 0)) AS mobile
) AS origins,
COUNT(DISTINCT root_page) AS origins,
NULL AS technologies
FROM (
SELECT
client,
COUNT(DISTINCT root_page) AS origins
FROM merged_pages
GROUP BY client
)
FROM merged_pages
GROUP BY client
`).postOps(ctx => `
SELECT
reports.run_export_job(
Expand Down