Skip to content

Commit 70bd03f

Browse files
devtofunmidevtofunmi
committed
Implemented web scraping fallback logic for previous match verification using BBC Sport.
1 parent 18bf5fb commit 70bd03f

4 files changed

Lines changed: 321 additions & 178 deletions

File tree

‎lib/history/scraper.ts‎

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
2+
import axios from 'axios';
3+
import * as cheerio from 'cheerio';
4+
import { FootballDataMatch } from './verifier';
5+
6+
/**
7+
* Scrapes BBC Sport for match results on a specific date.
8+
* Returns a list of standardized 'FootballDataMatch' objects for compatibility.
9+
*/
10+
11+
export async function scrapeBBCMatches(date: string): Promise<FootballDataMatch[]> {
12+
try {
13+
// BBC Date format: YYYY-MM-DD (e.g., 2026-01-01)
14+
const url = `https://www.bbc.com/sport/football/scores-fixtures/${date}`;
15+
console.log(`[Scraper] Fetching ${url}`);
16+
17+
const { data } = await axios.get(url, {
18+
headers: {
19+
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
20+
}
21+
});
22+
23+
const $ = cheerio.load(data);
24+
const matches: FootballDataMatch[] = [];
25+
26+
// BBC structure varies, but generally matches are in "group" containers or list items.
27+
// search for elements that contain team names and scores.
28+
// Identify semantic containers for matches.
29+
30+
// Strategy: Look for the specific "qa-match-block" or similar components.
31+
// As of 2025/26, BBC might use new classes. try generic selectors first.
32+
33+
// Select all match containers using the GridContainer class identified
34+
$('div[class*="GridContainer"]').each((_, el) => {
35+
const $el = $(el);
36+
37+
// Extract Teams
38+
const homeTeamName = $el.find('div[class*="TeamHome"] span[class*="DesktopValue"], [data-testid="home-team-name"]').first().text().trim();
39+
const awayTeamName = $el.find('div[class*="TeamAway"] span[class*="DesktopValue"], [data-testid="away-team-name"]').first().text().trim();
40+
41+
if (!homeTeamName || !awayTeamName) return;
42+
43+
// Extract Score
44+
const homeScoreText = $el.find('div[class*="HomeScore"], [data-testid="home-score"]').first().text().trim();
45+
const awayScoreText = $el.find('div[class*="AwayScore"], [data-testid="away-score"]').first().text().trim();
46+
47+
const homeScore = parseInt(homeScoreText);
48+
const awayScore = parseInt(awayScoreText);
49+
50+
// Match Status
51+
const statusText = $el.find('div[class*="StyledPeriod"], div[class*="Status"]').text().trim().toUpperCase();
52+
let status = 'SCHEDULED';
53+
54+
if (!isNaN(homeScore) && !isNaN(awayScore)) {
55+
status = 'FINISHED';
56+
if (statusText === 'FT' || statusText.includes('FINISHED')) status = 'FINISHED';
57+
else if (statusText.includes('LIVE') || statusText.includes('MINS')) status = 'IN_PLAY';
58+
}
59+
60+
const id = Math.abs((homeTeamName + awayTeamName).split('').reduce((a, b) => a = ((a << 5) - a) + b.charCodeAt(0) | 0, 0));
61+
62+
matches.push({
63+
id: id,
64+
utcDate: date,
65+
status: status,
66+
score: {
67+
winner: homeScore > awayScore ? 'HOME_TEAM' : awayScore > homeScore ? 'AWAY_TEAM' : 'DRAW',
68+
duration: 'REGULAR',
69+
fullTime: { home: isNaN(homeScore) ? null : homeScore, away: isNaN(awayScore) ? null : awayScore },
70+
halfTime: { home: null, away: null }
71+
},
72+
homeTeam: { id: 0, name: homeTeamName },
73+
awayTeam: { id: 0, name: awayTeamName }
74+
});
75+
});
76+
77+
// FALLBACK: Parse text-based "versus" patterns if DOM structure failed
78+
if (matches.length === 0) {
79+
console.log("[Scraper] DOM selectors failed, trying text pattern fallback...");
80+
// Look for "Team A versus Team B" patterns
81+
const text = $('body').text();
82+
// This is a very complex regex because BBC text is often jumbled in the DOM-to-text conversion
83+
84+
$('a, li, div').each((_, el) => {
85+
const elText = $(el).text();
86+
if (elText.includes(' versus ') && (elText.includes('FT') || elText.match(/\d+-\d+/))) {
87+
// Try to extract: "Home Team versus Away Team"
88+
const parts = elText.split(' versus ');
89+
if (parts.length >= 2) {
90+
const home = parts[0].trim().split(/\s{2,}/).pop() || ''; // Get last few words
91+
const rest = parts[1].trim();
92+
// Rest likely contains Away Team and Score
93+
// Example: "Watford FT1 - 2"
94+
const matchScore = rest.match(/(\d+)\s*-\s*(\d+)/) || rest.match(/FT\s*(\d+)\s*(\d+)/);
95+
if (matchScore) {
96+
const away = rest.split(matchScore[0])[0].trim();
97+
const hScore = parseInt(matchScore[1]);
98+
const aScore = parseInt(matchScore[2]);
99+
100+
if (home && away && !isNaN(hScore) && !isNaN(aScore)) {
101+
const id = Math.abs((home + away).split('').reduce((a, b) => a = ((a << 5) - a) + b.charCodeAt(0) | 0, 0));
102+
matches.push({
103+
id,
104+
utcDate: date,
105+
status: 'FINISHED',
106+
score: {
107+
winner: hScore > aScore ? 'HOME_TEAM' : aScore > hScore ? 'AWAY_TEAM' : 'DRAW',
108+
duration: 'REGULAR',
109+
fullTime: { home: hScore, away: aScore },
110+
halfTime: { home: null, away: null }
111+
},
112+
homeTeam: { id: 0, name: home },
113+
awayTeam: { id: 0, name: away }
114+
});
115+
}
116+
}
117+
}
118+
}
119+
});
120+
}
121+
122+
console.log(`[Scraper] Found ${matches.length} matches on BBC for ${date}`);
123+
return matches;
124+
125+
} catch (error: any) {
126+
console.error(`[Scraper] Failed to scrape ${date}: ${error.message}`);
127+
return [];
128+
}
129+
}

‎lib/history/verifier.ts‎

Lines changed: 76 additions & 134 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,9 @@
1-
import axios from 'axios';
21
import { HistoryItem } from './storage';
32

43
/**
54
* Football Data API match structure for type safety.
65
*/
7-
interface FootballDataMatch {
6+
export interface FootballDataMatch {
87
id: number;
98
utcDate: string;
109
status: string;
@@ -51,7 +50,15 @@ export function extractMatchId(id: string): number | null {
5150
*
5251
* Note: Contributors adding new bet types should add cases here.
5352
*/
54-
function verifyPrediction(
53+
/**
54+
* verifyPrediction
55+
*
56+
* The core mathematical engine that compares a prediction against result data.
57+
* Handles diverse markets like 1X2, Over/Under, BTTS, and Half-time specific bets.
58+
*
59+
* Note: Contributors adding new bet types should add cases here.
60+
*/
61+
export function verifyPrediction(
5562
prediction: string,
5663
homeGoals: number,
5764
awayGoals: number,
@@ -120,153 +127,88 @@ function verifyPrediction(
120127
}
121128

122129
/**
123-
* findMatchByTeams
124-
*
125-
* Fallback "Healer" logic. If a match ID is missing, searches the API for a match
126-
* between the two teams on or near the predicted date.
130+
* Searches a provided list of matches for a fuzzy string match.
127131
*/
128-
async function findMatchByTeams(
132+
export function findMatchInList(
129133
homeTeam: string,
130134
awayTeam: string,
131-
predictionDate: string | undefined, // The date we predicted the match for
132-
apiKey: string
133-
): Promise<number | null> {
134-
if (!predictionDate) return null;
135-
136-
try {
137-
const todayStr = new Date().toISOString().split('T')[0];
138-
const matchDate = new Date(predictionDate);
139-
140-
// Search from 3 days before predicted date up to today
141-
const dateFrom = new Date(matchDate);
142-
dateFrom.setDate(matchDate.getDate() - 3);
143-
144-
const fromStr = dateFrom.toISOString().split('T')[0];
145-
const toStr = todayStr; // Always search up to today to catch rescheduled matches
146-
147-
const response = await axios.get(`https://api.football-data.org/v4/matches`, {
148-
headers: { 'X-Auth-Token': apiKey },
149-
params: { dateFrom: fromStr, dateTo: toStr }
150-
});
151-
152-
const matches = response.data.matches as FootballDataMatch[];
153-
const totalFound = matches?.length || 0;
154-
155-
// DEBUG: Output all fixtures in the range for deep diagnostics
156-
if (totalFound > 0) {
157-
const pool = matches.map(m => `${m.id}:${m.homeTeam.name} vs ${m.awayTeam.name}`).join(' | ');
158-
console.info(`[Verifier] Searching ${homeTeam} vs ${awayTeam} in range ${fromStr} to ${toStr}. Pool (${totalFound} matches): ${pool}`);
159-
}
160-
161-
const normalize = (n: string) => n.toLowerCase()
162-
.replace(/\b(fc|afc|cf|sc|ac|united|city|rovers|albion|town|athletic|clube de|club|de|as|ss|ssc|bc|uc|us|cd|cuba|futebol|sad|sports|sporting|international|internazionale|italy|portugal|spain|france|england|germany)\b/g, '')
135+
matches: FootballDataMatch[]
136+
): FootballDataMatch | null {
137+
const normalize = (n: string) => {
138+
if (!n) return '';
139+
return n.toLowerCase()
140+
.replace(/\b(fc|afc|cf|sc|ac|rovers|albion|town|athletic|clube de|club|de|as|ss|ssc|bc|uc|us|cd|cuba|futebol|sad|sports|sporting|international|internazionale|italy|portugal|spain|france|england|germany)\b/g, '')
163141
.replace(/[\W_]+/g, ' ')
164142
.trim();
143+
};
165144

166-
// Technical Mappings
167-
const nicknames: Record<string, string[]> = {
168-
'wolves': ['wolverhampton'],
169-
'inter': ['internazionale'],
170-
'mancity': ['manchester city'],
171-
'manutd': ['manchester united'],
172-
'avs': ['avs futebol sad'],
173-
'porto': ['fc porto', 'futebol clube do porto']
174-
};
175-
176-
const getSearchTerms = (name: string) => {
177-
const n = normalize(name);
178-
const terms = new Set([n]);
179-
// Extract key words (e.g. "Porto" from "FC Porto")
180-
n.split(' ').forEach(w => { if (w.length > 3) terms.add(w); });
181-
Object.entries(nicknames).forEach(([key, val]) => {
182-
if (n.includes(key) || key.includes(n)) val.forEach(v => terms.add(v));
183-
});
184-
return Array.from(terms);
185-
};
186-
187-
const targetHTerms = getSearchTerms(homeTeam);
188-
const targetATerms = getSearchTerms(awayTeam);
145+
const nicknames: Record<string, string[]> = {
146+
'wolves': ['wolverhampton', 'wolverhampton wanderers'],
147+
'inter': ['internazionale', 'inter milan', 'internazionale milano'],
148+
'mancity': ['manchester city'],
149+
'manutd': ['manchester united', 'man utd'],
150+
'spurs': ['tottenham', 'tottenham hotspur'],
151+
'avs': ['avs futebol sad'],
152+
'porto': ['fc porto', 'futebol clube do porto'],
153+
'benfica': ['sl benfica', 'sport lisboa e benfica'],
154+
'milan': ['ac milan'],
155+
'verona': ['hellas verona']
156+
};
189157

190-
const match = matches.find(m => {
191-
const h = normalize(m.homeTeam.name);
192-
const a = normalize(m.awayTeam.name);
158+
const getSearchTerms = (name: string) => {
159+
const n = normalize(name);
160+
const terms = new Set([n]);
161+
// Extract key words (e.g. "Porto" from "FC Porto")
162+
n.split(' ').forEach(w => { if (w.length > 3) terms.add(w); });
163+
Object.entries(nicknames).forEach(([key, val]) => {
164+
if (n.includes(key) || key.includes(n)) val.forEach(v => terms.add(v));
165+
});
166+
return Array.from(terms);
167+
};
193168

194-
// Stricter matching: ONE team from prediction MUST match Home API,
195-
// AND the OTHER team from prediction MUST match Away API.
196-
const homeMatchesH = targetHTerms.some(t => h.includes(t) || t.includes(h));
197-
const homeMatchesA = targetHTerms.some(t => a.includes(t) || t.includes(a));
169+
const targetHTerms = getSearchTerms(homeTeam);
170+
const targetATerms = getSearchTerms(awayTeam);
198171

199-
const awayMatchesH = targetATerms.some(t => h.includes(t) || t.includes(h));
200-
const awayMatchesA = targetATerms.some(t => a.includes(t) || t.includes(a));
172+
return matches.find(m => {
173+
const h = normalize(m.homeTeam.name);
174+
const a = normalize(m.awayTeam.name);
201175

202-
const normalOrder = homeMatchesH && awayMatchesA;
203-
const swappedOrder = homeMatchesA && awayMatchesH;
176+
const homeMatchesH = targetHTerms.some(t => h.includes(t) || t.includes(h));
177+
const homeMatchesA = targetHTerms.some(t => a.includes(t) || t.includes(a));
178+
const awayMatchesH = targetATerms.some(t => h.includes(t) || t.includes(h));
179+
const awayMatchesA = targetATerms.some(t => a.includes(t) || t.includes(a));
204180

205-
return normalOrder || swappedOrder;
206-
});
181+
const normalOrder = homeMatchesH && awayMatchesA;
182+
const swappedOrder = homeMatchesA && awayMatchesH;
207183

208-
if (match) {
209-
console.info(`[Verifier] SUCCESS: Found ${match.homeTeam.name} vs ${match.awayTeam.name} for ${homeTeam} vs ${awayTeam}`);
210-
return match.id;
211-
} else {
212-
console.warn(`[Verifier] FAILED: No match found for ${homeTeam} vs ${awayTeam}`);
213-
return null;
214-
}
215-
} catch (e: any) {
216-
if (e.response?.status === 429) throw e;
217-
console.error("[Verifier] API Error during search:", homeTeam, "vs", awayTeam);
218-
return null;
219-
}
184+
return normalOrder || swappedOrder;
185+
}) || null;
220186
}
221187

222188
/**
223-
* verifyMatch
224-
*
225-
* Main orchestration for match verification. Retrieves match status/results and
226-
* settles predictions.
189+
* Verifies a prediction using a provided Match object (avoids API calls)
227190
*/
228-
export async function verifyMatch(item: HistoryItem, apiKey: string, date?: string): Promise<HistoryItem> {
229-
let matchId: number | null = item.matchId || extractMatchId(item.id || '');
230-
231-
// Trigger Healer if extraction fails (older "local" IDs)
232-
if (!matchId) {
233-
matchId = await findMatchByTeams(item.homeTeam, item.awayTeam, date, apiKey);
191+
export function verifyMatchFromData(item: HistoryItem, match: FootballDataMatch): HistoryItem {
192+
if (match.status !== 'FINISHED' && match.status !== 'IN_PLAY' && match.status !== 'PAUSED') {
193+
return { ...item, result: 'Pending', matchId: match.id };
234194
}
235195

236-
if (!matchId) return { ...item, result: 'Pending' };
237-
238-
try {
239-
const response = await axios.get(`https://api.football-data.org/v4/matches/${matchId}`, {
240-
headers: { 'X-Auth-Token': apiKey }
241-
});
242-
243-
const match = response.data as FootballDataMatch;
244-
245-
// Skip verification if match hasn't started or isn't finished enough to be certain
246-
if (match.status !== 'FINISHED' && match.status !== 'IN_PLAY' && match.status !== 'PAUSED') {
247-
return { ...item, result: 'Pending', matchId };
248-
}
249-
250-
const { home, away } = match.score.fullTime;
251-
if (home === null || away === null) return { ...item, result: 'Pending', matchId };
252-
253-
const result = verifyPrediction(
254-
item.prediction,
255-
home,
256-
away,
257-
match.score.halfTime.home,
258-
match.score.halfTime.away
259-
);
260-
261-
return {
262-
...item,
263-
result: match.status === 'FINISHED' ? result : 'Pending',
264-
score: formatScore(match),
265-
matchId
266-
};
196+
const { home, away } = match.score.fullTime;
197+
if (home === null || away === null) return { ...item, result: 'Pending', matchId: match.id };
198+
199+
const result = verifyPrediction(
200+
item.prediction,
201+
home,
202+
away,
203+
match.score.halfTime.home,
204+
match.score.halfTime.away
205+
);
206+
207+
return {
208+
...item,
209+
result: match.status === 'FINISHED' ? result : 'Pending',
210+
score: formatScore(match),
211+
matchId: match.id
212+
};
213+
}
267214

268-
} catch (error: any) {
269-
if (error.response?.status === 429) throw error;
270-
return { ...item, matchId };
271-
}
272-
}

0 commit comments

Comments
 (0)