Skip to content

Commit 4cd6bb0

Browse files
EwoutHEwoutH
authored
KiM: Improve metadata parsing (#3631)
1 parent 292efb8 commit 4cd6bb0

1 file changed

Lines changed: 186 additions & 38 deletions

File tree

‎KiM.js‎

Lines changed: 186 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
"inRepository": true,
1010
"translatorType": 4,
1111
"browserSupport": "gcsibv",
12-
"lastUpdated": "2026-01-02 16:28:12"
12+
"lastUpdated": "2026-07-15 13:55:27"
1313
}
1414

1515
/*
@@ -88,8 +88,44 @@ async function scrape(doc, url = doc.location.href) {
8888
// Detect language from URL for proper institution name
8989
var isEnglish = url.includes('english.kimnet.nl');
9090

91-
// Title from the page heading
92-
item.title = text(doc, 'h1.nav-bar__page-title');
91+
// Parse #elastic-content (title, author, date)
92+
let elastic = {};
93+
let elasticEl = doc.querySelector('#elastic-content');
94+
if (elasticEl) {
95+
try {
96+
elastic = JSON.parse(elasticEl.textContent);
97+
}
98+
catch (e) {
99+
Zotero.debug('Failed to parse elastic-content: ' + e);
100+
}
101+
}
102+
103+
// Parse JSON-LD, selecting the WebPage node specifically
104+
// (the page contains multiple ld+json blocks in unspecified order)
105+
let webPage = {};
106+
for (let script of doc.querySelectorAll('script[type="application/ld+json"]')) {
107+
try {
108+
let data = JSON.parse(script.textContent);
109+
let graph = data['@graph'] || [data];
110+
let page = graph.find(n => n['@type'] === 'WebPage');
111+
if (page) {
112+
webPage = page;
113+
break;
114+
}
115+
}
116+
catch (e) {
117+
Zotero.debug('Failed to parse JSON-LD: ' + e);
118+
}
119+
}
120+
121+
// Title: prefer JSON-LD, fall back to elastic, then to page heading
122+
item.title = webPage.name || elastic.pageTitle || text(doc, 'h1.nav-bar__page-title');
123+
124+
// Date: prefer datePublished, fall back to publicationDate; keep date only
125+
let rawDate = webPage.datePublished || elastic.publicationDate;
126+
if (rawDate) {
127+
item.date = ZU.strToISO(rawDate);
128+
}
93129

94130
// Abstract from the intro section
95131
item.abstractNote = text(doc, '.intro .rich-text');
@@ -106,45 +142,24 @@ async function scrape(doc, url = doc.location.href) {
106142
}
107143
item.place = 'Den Haag';
108144

109-
// Language tag
110-
item.language = doc.documentElement.lang || (isEnglish ? 'en' : 'nl');
111-
112-
// Process downloads to get metadata and attachments
113-
var downloads = doc.querySelectorAll('.download-list__item');
114-
115-
for (let download of downloads) {
116-
let metadata = download.querySelector('.meta-data');
117-
if (!metadata) continue;
118-
119-
let metaParts = Array.from(metadata.querySelectorAll('span')).map(s => s.textContent.trim());
145+
// Language
146+
item.language = webPage.inLanguage || doc.documentElement.lang || (isEnglish ? 'en' : 'nl');
120147

121-
// Extract date (format: DD-MM-YYYY)
122-
let dateStr = metaParts.find(p => /^\d{2}-\d{2}-\d{4}$/.test(p));
123-
if (dateStr && !item.date) {
124-
// Convert from DD-MM-YYYY to YYYY-MM-DD
125-
let parts = dateStr.split('-');
126-
item.date = `${parts[2]}-${parts[1]}-${parts[0]}`;
127-
}
128-
129-
// Extract authors - look for entries with commas that aren't file sizes or page counts
130-
let authors = metaParts.find(p => p.includes(',')
131-
&& !/\d/.test(p.split(',')[0]) // First part shouldn't contain numbers
132-
&& !p.includes('pagina')
133-
&& !p.includes('pages')
134-
&& !p.includes('KB')
135-
&& !p.includes('MB')
136-
);
137-
138-
if (authors && !item.creators.length) {
139-
// Split on comma and add each author
140-
let authorList = authors.split(',').map(a => a.trim());
141-
for (let author of authorList) {
142-
if (!author) continue;
143-
item.creators.push(ZU.cleanAuthor(author, 'author', false));
148+
// Authors from elastic-content (English pages have this; Dutch typically don't)
149+
// Assumes "Firstname Lastname, Firstname Lastname" — a comma separates
150+
// distinct authors, NOT a single "Lastname, Firstname" entry.
151+
if (elastic.author) {
152+
for (let name of elastic.author.split(',')) {
153+
name = name.trim();
154+
if (name) {
155+
item.creators.push(ZU.cleanAuthor(name, 'author', false));
144156
}
145157
}
158+
}
146159

147-
// Get PDF attachment
160+
// PDF attachments from the download list
161+
var downloads = doc.querySelectorAll('.download-list__item');
162+
for (let download of downloads) {
148163
let pdfLink = download.querySelector('a[href$=".pdf"]');
149164
if (pdfLink) {
150165
let pdfTitle = text(download, '.title');
@@ -254,6 +269,139 @@ var testCases = [
254269
"seeAlso": []
255270
}
256271
]
272+
},
273+
{
274+
"type": "web",
275+
"url": "https://english.kimnet.nl/documents/2026/05/21/energy-saving-in-transport-through-avoid-shift-policy-measures",
276+
"items": [
277+
{
278+
"itemType": "report",
279+
"title": "Energy saving in transport through avoid/shift policy measures",
280+
"creators": [
281+
{
282+
"firstName": "Stefan",
283+
"lastName": "Bakker",
284+
"creatorType": "author"
285+
},
286+
{
287+
"firstName": "Saeda",
288+
"lastName": "Moorman",
289+
"creatorType": "author"
290+
}
291+
],
292+
"date": "2026-05-21",
293+
"abstractNote": "Energy demand reduction in the transport sector can be achieved by reducing transport demand (avoid), encouraging the use of alternative modes of transport (shift), and improving vehicle energy efficiency (improve). Research by the KiM Netherlands Institute for Transport Policy Analysis shows that, from a policy perspective, it is difficult to achieve substantial energy savings using the first two strategies (avoid and shift). The importance of energy savings has increased due to geopolitical developments and in light of energy supply security.",
294+
"institution": "Netherlands Institute for Transport Policy Analysis",
295+
"language": "en",
296+
"libraryCatalog": "KiM",
297+
"place": "Den Haag",
298+
"url": "https://english.kimnet.nl/documents/2026/05/21/energy-saving-in-transport-through-avoid-shift-policy-measures",
299+
"attachments": [
300+
{
301+
"title": "Brochure - Energy saving in transport through avoid/shift policy measures",
302+
"mimeType": "application/pdf"
303+
},
304+
{
305+
"title": "Background report - Energy saving in transport through avoid/shift policy measures: an exploratory study into dealing with scarcity",
306+
"mimeType": "application/pdf"
307+
},
308+
{
309+
"title": "Snapshot",
310+
"mimeType": "text/html"
311+
}
312+
],
313+
"tags": [],
314+
"notes": [],
315+
"seeAlso": []
316+
}
317+
]
318+
},
319+
{
320+
"type": "web",
321+
"url": "https://www.kimnet.nl/documenten/2026/05/28/bedrijfseconomische-kostenkengetallen-voor-het-goederenvervoer-update-2026",
322+
"items": [
323+
{
324+
"itemType": "report",
325+
"title": "Bedrijfseconomische kostenkengetallen voor het goederenvervoer, update 2026",
326+
"creators": [],
327+
"date": "2026-05-28",
328+
"abstractNote": "Om vraagstukken waarvoor bedrijfseconomische kostenkengetallen van het goederenvervoer nodig zijn te kunnen beantwoorden - bijvoorbeeld over de impact van stijgende brandstofkosten op de totale vervoerskosten - is het noodzakelijk om actuele kostengegevens te hebben. Vanuit eerdere edities van dit onderzoek waren kostenkengetallen beschikbaar voor de jaren 2016-2021. Het Kennisinstituut voor Mobiliteitsbeleid (KiM) heeft de kostenkengetallen voor het goederenvervoer nu geactualiseerd tot en met 2024. De kostenkengetallen zijn beschikbaar voor de vervoerwijzen weg, spoor, binnenvaart, zeevaart, luchtvaart, en buisleiding. Kostenkengetallen voor buisleidingen zijn voor het eerst ontwikkeld in deze editie. Het KiM heeft de kostenkengetallen laten verzamelen door de onderzoeksbureaus Panteia en Ecorys.",
329+
"institution": "Kennisinstituut voor Mobiliteitsbeleid",
330+
"language": "nl",
331+
"libraryCatalog": "KiM",
332+
"place": "Den Haag",
333+
"url": "https://www.kimnet.nl/documenten/2026/05/28/bedrijfseconomische-kostenkengetallen-voor-het-goederenvervoer-update-2026",
334+
"attachments": [
335+
{
336+
"title": "Notitie - Bedrijfseconomische kostenkengetallen voor het goederenvervoer, update 2026",
337+
"mimeType": "application/pdf"
338+
},
339+
{
340+
"title": "Achtergrondrapport Panteia - Bedrijfseconomische kostenkengetallen voor het goederenvervoer",
341+
"mimeType": "application/pdf"
342+
},
343+
{
344+
"title": "Achtergrondrapport Ecorys - Verkenning bedrijfseconomische kosten van transport via buisleidingen",
345+
"mimeType": "application/pdf"
346+
},
347+
{
348+
"title": "Tabel 1 - Kostenkengetallen van de binnenvaart",
349+
"mimeType": "application/pdf"
350+
},
351+
{
352+
"title": "Tabel 2 - Kostenkengetallen van de binnenvaart verschijningsvorm",
353+
"mimeType": "application/pdf"
354+
},
355+
{
356+
"title": "Tabel 3 - Kostenkengetallen buisleiding",
357+
"mimeType": "application/pdf"
358+
},
359+
{
360+
"title": "Tabel 4 - Kostenkengetallen van de luchtvaart",
361+
"mimeType": "application/pdf"
362+
},
363+
{
364+
"title": "Tabel 5 - Kostenkengetallen van de luchtvaart verschijningsvorm",
365+
"mimeType": "application/pdf"
366+
},
367+
{
368+
"title": "Tabel 6 - Kostenkengetallen overslag containers",
369+
"mimeType": "application/pdf"
370+
},
371+
{
372+
"title": "Tabel 7 - Kostenkengetallen van het spoorvervoer",
373+
"mimeType": "application/pdf"
374+
},
375+
{
376+
"title": "Tabel 8 - Kostenkengetallen van het spoorvervoer verschijningsvorm",
377+
"mimeType": "application/pdf"
378+
},
379+
{
380+
"title": "Tabel 9 - Kostenkengetallen van het wegvervoer",
381+
"mimeType": "application/pdf"
382+
},
383+
{
384+
"title": "Tabel 10 - Kostenkengetallen van het wegvervoer verschijningsvorm",
385+
"mimeType": "application/pdf"
386+
},
387+
{
388+
"title": "Tabel 11 - Kostenkengetallen van de zeevaart",
389+
"mimeType": "application/pdf"
390+
},
391+
{
392+
"title": "Tabel 12 - Kostenkengetallen van de zeevaart verschijningsvorm",
393+
"mimeType": "application/pdf"
394+
},
395+
{
396+
"title": "Snapshot",
397+
"mimeType": "text/html"
398+
}
399+
],
400+
"tags": [],
401+
"notes": [],
402+
"seeAlso": []
403+
}
404+
]
257405
}
258406
]
259407
/** END TEST CASES **/

0 commit comments

Comments
 (0)