Skip to content

Commit 9ce9af8

Browse files
03c03c
authored
Merge pull request #95 from ElixirTeSS/fix_portugal_scraper
Fix Portugal scraper
2 parents 2ba3812 + 53b174a commit 9ce9af8

6 files changed

Lines changed: 82 additions & 54 deletions

File tree

‎Gemfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ source 'https://rubygems.org'
22

33
gem 'geocoder'
44
gem 'inifile'
5-
gem 'nokogiri', '~> 1.8.1'
5+
gem 'nokogiri', '~> 1.10.9'
66
gem 'redcarpet'
77
gem 'simple-rss'
88
gem 'google_places'

‎Gemfile.lock‎

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -19,11 +19,12 @@ GIT
1919
GEM
2020
remote: https://rubygems.org/
2121
specs:
22-
activesupport (5.2.1)
22+
activesupport (6.0.3.1)
2323
concurrent-ruby (~> 1.0, >= 1.0.2)
2424
i18n (>= 0.7, < 2)
2525
minitest (~> 5.1)
2626
tzinfo (~> 1.1)
27+
zeitwerk (~> 2.2, >= 2.2.2)
2728
addressable (2.7.0)
2829
public_suffix (>= 2.0.2, < 5.0)
2930
bcp47 (0.3.3)
@@ -52,7 +53,7 @@ GEM
5253
domain_name (~> 0.5)
5354
httparty (0.14.0)
5455
multi_xml (>= 0.5.2)
55-
i18n (1.5.1)
56+
i18n (1.8.2)
5657
concurrent-ruby (~> 1.0)
5758
icalendar (2.4.1)
5859
inifile (3.0.0)
@@ -98,17 +99,17 @@ GEM
9899
mime-types (3.2.2)
99100
mime-types-data (~> 3.2015)
100101
mime-types-data (3.2018.0812)
101-
mini_portile2 (2.3.0)
102-
minitest (5.11.3)
102+
mini_portile2 (2.4.0)
103+
minitest (5.14.1)
103104
multi_json (1.14.1)
104105
multi_xml (0.5.5)
105106
net (0.3.3)
106107
activesupport
107108
net-http-persistent (3.1.0)
108109
connection_pool (~> 2.2)
109110
netrc (0.11.0)
110-
nokogiri (1.8.5)
111-
mini_portile2 (~> 2.3.0)
111+
nokogiri (1.10.9)
112+
mini_portile2 (~> 2.4.0)
112113
nokogumbo (1.5.0)
113114
nokogiri
114115
public_suffix (3.1.1)
@@ -201,11 +202,12 @@ GEM
201202
temple (0.8.2)
202203
thread_safe (0.3.6)
203204
tilt (2.0.10)
204-
tzinfo (1.2.5)
205+
tzinfo (1.2.7)
205206
thread_safe (~> 0.1)
206207
unf (0.1.4)
207208
unf_ext
208209
unf_ext (0.0.7.5)
210+
zeitwerk (2.3.0)
209211

210212
PLATFORMS
211213
ruby
@@ -217,7 +219,7 @@ DEPENDENCIES
217219
icalendar
218220
inifile
219221
linkeddata (~> 2.0)
220-
nokogiri (~> 1.8.1)
222+
nokogiri (~> 1.10.9)
221223
redcarpet
222224
sanitize
223225
simple-rss

‎app/scrapers/edinburgh_scraper.rb‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ def scrape
3434
json = json[1].gsub('<!--//--><![CDATA[// ><!--', '')
3535
json = json.gsub('//--><!]]>', '')
3636
json = json.gsub('<br />', '')
37+
json = json.gsub("\n", "")
3738
event = Tess::Rdf::EventExtractor.new(json, :jsonld).extract { |p| Tess::API::Event.new(p) }.first
3839
event.content_provider = cp
3940
event.event_types = [:workshops_and_courses]

‎app/scrapers/intermine_scraper.rb‎

Lines changed: 55 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1,40 +1,64 @@
1-
21
class IntermineScraper < Tess::Scrapers::Scraper
3-
42
def self.config
53
{
6-
name: 'Intermine Scraper',
7-
index_page: 'http://intermine.org/tutorials/'
4+
name: "Intermine Scraper",
5+
index_page: "http://intermine.org/tutorials/",
86
}
97
end
108

119
def scrape
12-
cp = add_content_provider(Tess::API::ContentProvider.new(
13-
title: "InterMine", #name
14-
url: "http://intermine.org", #url
15-
image_url: "https://cdn.rawgit.com/intermine/design-materials/ff1ec6bf/logos/intermine/intermine-300x37.png", #logo
16-
description: "nterMine integrates biological data sources, making it easy to query and analyse data.", #description
17-
content_provider_type: :project,
18-
node_name: :'UK'
19-
))
10+
cp = add_content_provider(Tess::API::ContentProvider.new(
11+
title: "InterMine", #name
12+
url: "http://intermine.org", #url
13+
image_url: "https://cdn.rawgit.com/intermine/design-materials/ff1ec6bf/logos/intermine/intermine-300x37.png", #logo
14+
description: "nterMine integrates biological data sources, making it easy to query and analyse data.", #description
15+
content_provider_type: :project,
16+
node_name: :'UK',
17+
))
18+
19+
#page = Nokogiri::HTML.parse(open(config[:index_page]).read)
20+
index = "http://intermine.org/tutorials/"
21+
page = Nokogiri::HTML.parse(open(index).read)
22+
tutorials = page.xpath("/html/body/main/ul/li")
2023

21-
#page = Nokogiri::HTML.parse(open(config[:index_page]).read)
22-
index = 'http://intermine.org/tutorials/'
23-
page = Nokogiri::HTML.parse(open(index).read)
24-
tutorials = page.xpath('/html/body/main/ul/li')
25-
26-
tutorials.each do |tutorial|
27-
url = tutorial.children.first.attributes['href'].value
28-
title = tutorial.text.split('-').first
29-
description = tutorial.text
30-
event = Tess::API::Material.new(
31-
title: title,
32-
url: url,
33-
content_provider: cp,
34-
short_description: description,
35-
keywords: title.split(' ').first
36-
)
37-
add_material(event)
38-
end
39-
end
24+
tutorials.each do |tutorial|
25+
if !tutorial.children.first.attributes["href"].nil?
26+
url = tutorial.children.first.attributes["href"].value
27+
title = tutorial.text.split("-").first
28+
description = tutorial.text
29+
event = Tess::API::Material.new(
30+
title: title,
31+
url: url,
32+
content_provider: cp,
33+
short_description: description,
34+
keywords: title.split(" ").first,
35+
)
36+
add_material(event)
37+
end
38+
end
39+
40+
#page = Nokogiri::HTML.parse(open(config[:index_page]).read)
41+
index = "http://intermine.org/training-workshops/"
42+
page = Nokogiri::HTML.parse(open(index).read)
43+
tutorials = page.xpath("/html/body/div/section/ul/li/a")
44+
tutorials.each do |tutorial|
45+
if !tutorial.attributes["href"].nil?
46+
url = tutorial.attributes["href"].value
47+
# Complete relative URL paths
48+
if !url.include? "http"
49+
url = index + url
50+
end
51+
title = tutorial.children.text
52+
description = title
53+
event = Tess::API::Material.new(
54+
title: title,
55+
url: url,
56+
content_provider: cp,
57+
short_description: description,
58+
keywords: title.split(" ").first,
59+
)
60+
add_material(event)
61+
end
62+
end
63+
end
4064
end

‎app/scrapers/portugal_events_scraper.rb‎

Lines changed: 14 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -5,15 +5,16 @@ class PortugalEventsScraper < Tess::Scrapers::Scraper
55
def self.config
66
{
77
name: 'ELIXIR Portugal',
8-
root_url: 'http://elixir-portugal.org/events',
9-
base_url: 'http://elixir-portugal.org'
8+
# TODO: change URL to https://biodata.pt/events before deploying
9+
root_url: 'https://biodata.pt/past-events',
10+
base_url: 'https://biodata.pt'
1011
}
1112
end
1213

1314
def scrape
1415
cp = add_content_provider(Tess::API::ContentProvider.new(
1516
{ title: "ELIXIR Portugal",
16-
url: "https://elixir-portugal.org",
17+
url: "https://biodata.pt",
1718
image_url: "https://tess.elixir-europe.org/assets/nodes/logos/PT-9f2611b1953f3109fa81668d960e7068390f4ef69be8f4b950ec0e8d7b106503.png",
1819
description: "ELIXIR Portugal is organized as a consortium of Portuguese research institutions which are part of the national biological information network, BioData.pt. Like ELIXIR itself, the Portuguese node is a decentralized network of specialized centers, under a common hardware and software infrastructure, and with shared training and industry/entrepreneurship programmes.",
1920
content_provider_type: :organisation,
@@ -22,21 +23,21 @@ def scrape
2223

2324

2425

25-
index = Nokogiri::HTML(open(config[:root_url]))
26-
urls = index.search('tbody a').map{|x| x.values}.flatten
26+
index = Nokogiri::HTML(open(config[:root_url]))
27+
urls = index.search('tbody td.views-field-title a').map{|x| x.values}.flatten
2728

2829
urls.each do |url|
29-
#url =
30-
html = open(config[:base_url] + url).read
31-
#extract JSON with regex because passing whole JSON to RDFEventExtractor throws errors up.
32-
a = /<script type="application\/ld\+json">(.*)<\/script>/m.match(html)
30+
#url =
31+
html = open(config[:base_url] + url).read
32+
#extract JSON with regex because passing whole JSON to RDFEventExtractor throws errors up.
33+
a = /<script type="application\/ld\+json">(.*?)<\/script>/m.match(html)
3334

3435
if a
35-
events = Tess::Rdf::EventExtractor.new(a[1], :jsonld).extract { |p| Tess::API::Event.new(p) }
36+
events = Tess::Rdf::EventExtractor.new(a[1], :jsonld).extract { |p| Tess::API::Event.new(p) }
3637

37-
events.each do |event|
38-
event.content_provider = cp
39-
add_event(event)
38+
events.each do |event|
39+
event.content_provider = cp
40+
add_event(event)
4041
end
4142
end
4243
end

‎run_scrapers.rb‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@
4545
NbisEventsScraper,
4646
NgsRegistryScraper,
4747
OpenTargetJsonScraper,
48-
PortugalEventsScraper,
48+
#PortugalEventsScraper,
4949
PraceEventsScraper,
5050
RssScraper,
5151
SheffieldScraper,

0 commit comments

Comments
 (0)