Skip to content

Commit fc43f45

Browse files
committed
Rewrite parser for new instance list fmt
iv-org/documentation#76
1 parent c4da7f3 commit fc43f45

File tree

4 files changed

+183
-58
lines changed

4 files changed

+183
-58
lines changed

src/fetch.cr

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
# Fetch a country's emoji flag from their country code (ISO 3166 alpha-2).
2+
#
3+
# A flag is made out of two regional indicator symbols.
4+
# So in order to convert from an ISO 3166 alpha-2 code into unicode we'll have to
5+
# add a specific offset to each character to make them into the required regional
6+
# indicator symbols. This offset is exactly 0x1f1a5.
7+
#
8+
# Reference implementation https://schinckel.net/2015/10/29/unicode-flags-in-python/
9+
private def fetch_flag(country_code)
10+
return country_code.codepoints.map { |codepoint| (codepoint + 0x1f1a5).chr }.join("")
11+
end
12+
13+
# Extracts the nested modified information containing source url and changes.
14+
private def extract_modified_information(modified_hash)
15+
if modified = modified_hash.as_h?
16+
return Modified.new(
17+
source: modified["source"].as_s,
18+
changes: modified["changes"].as_s
19+
)
20+
end
21+
end
22+
23+
# Extracts information common to all instance types.
24+
private def extract_prerequisites(instance_data)
25+
uri = URI.parse(instance_data["url"].to_s)
26+
host = uri.host
27+
28+
# Fetch country data
29+
region = instance_data["country"].to_s
30+
flag = fetch_flag(region)
31+
32+
privacy_policy = instance_data["privacy_policy"].as_s?
33+
owner = {name: instance_data["owner"].to_s.split("/")[-1].to_s, url: instance_data["owner"].as_s}
34+
modified = extract_modified_information(instance_data["modified"])
35+
notes = instance_data["notes"].as_a?
36+
37+
mirrors = [] of Mirrors
38+
instance_data["mirrors"].as_a?.try &.each do |m|
39+
mirrors << Mirrors.new(
40+
url: m["url"].as_s,
41+
region: m["country"].as_s,
42+
flag: fetch_flag(m["country"].as_s)
43+
)
44+
end
45+
46+
return uri, host, region, flag, privacy_policy, owner, modified, notes, mirrors
47+
end
48+
49+
def prepare_http_instance(instance_data, instances_storage, monitors)
50+
uri, host, region, flag, privacy_policy, owner, modified, notes, mirrors = extract_prerequisites(instance_data)
51+
52+
# Fetch status information
53+
if status = instance_data["status"].as_h?
54+
status_url = status["url"].as_s
55+
else
56+
status_url = nil
57+
end
58+
59+
ddos_mitm_protection = instance_data["ddos_mitm_protection"].as_s?
60+
61+
client = HTTP::Client.new(uri)
62+
client.connect_timeout = 5.seconds
63+
client.read_timeout = 5.seconds
64+
65+
begin
66+
stats = JSON.parse(client.get("/api/v1/stats").body)
67+
rescue ex
68+
stats = nil
69+
end
70+
71+
monitor = monitors.try &.select { |monitor| monitor["name"].try &.as_s == host }[0]?
72+
return {
73+
region: region,
74+
flag: flag,
75+
stats: stats,
76+
type: "https",
77+
uri: uri.to_s,
78+
status_url: status_url,
79+
privacy_policy: privacy_policy,
80+
ddos_mitm_protection: ddos_mitm_protection,
81+
owner: owner,
82+
modified: modified,
83+
mirrors: mirrors,
84+
notes: notes,
85+
monitor: monitor || instances_storage[host]?.try &.[:monitor]?,
86+
}
87+
end
88+
89+
def prepare_onion_instance(instance_data, instances_storage)
90+
uri, host, region, flag, privacy_policy, owner, modified, notes, mirrors = extract_prerequisites(instance_data)
91+
92+
associated_clearnet_instance = instance_data["associated_clearnet_instance"].as_s?
93+
94+
if CONFIG["fetch_onion_instance_stats"]?
95+
begin
96+
args = Process.parse_arguments("--socks5-hostname '#{CONFIG["tor_sock_proxy_address"]}:#{CONFIG["tor_sock_proxy_port"]}' 'http://#{uri.host}/api/v1/stats'")
97+
response = nil
98+
Process.run("curl", args: args) do |result|
99+
data = result.output.read_line
100+
response = JSON.parse(data)
101+
end
102+
103+
stats = response
104+
rescue ex
105+
stats = nil
106+
end
107+
else
108+
stats = nil
109+
end
110+
111+
return {
112+
region: region,
113+
flag: flag,
114+
stats: stats,
115+
type: "onion",
116+
uri: uri.to_s,
117+
associated_clearnet_instance: associated_clearnet_instance,
118+
privacy_policy: privacy_policy,
119+
owner: owner,
120+
modified: modified,
121+
mirrors: mirrors,
122+
notes: notes,
123+
monitor: nil,
124+
}
125+
end

src/helpers/helpers.cr

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
require "yaml"
22

33
def load_config
4-
config = YAML.parse(File.read("config.yml"))
5-
return config
4+
return YAML.parse(File.read("config.yml"))
5+
end
6+
7+
def load_instance_yaml(contents)
8+
return YAML.parse(contents)
69
end

src/instances.cr

Lines changed: 52 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ require "http/client"
1818
require "kemal"
1919
require "uri"
2020

21+
require "./fetch.cr"
2122
require "./helpers/*"
2223

2324
CONFIG = load_config()
@@ -28,9 +29,41 @@ macro rendered(filename)
2829
render "src/instances/views/#{{{filename}}}.ecr"
2930
end
3031

31-
alias Instance = NamedTuple(flag: String?, region: String?, stats: JSON::Any?, type: String, uri: String, monitor: JSON::Any?)
32-
33-
INSTANCES = {} of String => Instance
32+
# Nested data within instances
33+
alias Owner = NamedTuple(name: String, url: String)
34+
alias Modified = NamedTuple(source: String, changes: String)
35+
alias Mirrors = NamedTuple(url: String, region: String, flag: String)
36+
37+
alias ClearNetInstance = NamedTuple(
38+
flag: String,
39+
region: String,
40+
stats: JSON::Any?,
41+
type: String,
42+
uri: String,
43+
status_url: String?,
44+
privacy_policy: String?,
45+
ddos_mitm_protection: String?,
46+
owner: Owner,
47+
modified: Modified?,
48+
mirrors: Array(Mirrors)?,
49+
notes: Array(YAML::Any)?,
50+
monitor: JSON::Any?)
51+
52+
alias OnionInstance = NamedTuple(
53+
flag: String,
54+
region: String,
55+
stats: JSON::Any?,
56+
type: String,
57+
uri: String,
58+
associated_clearnet_instance: String?,
59+
privacy_policy: String?,
60+
owner: Owner,
61+
modified: Modified?,
62+
mirrors: Array(Mirrors)?,
63+
notes: Array(YAML::Any)?,
64+
monitor: JSON::Any?)
65+
66+
INSTANCES = {} of String => ClearNetInstance | OnionInstance
3467

3568
spawn do
3669
loop do
@@ -54,59 +87,23 @@ spawn do
5487
break
5588
end
5689
end
90+
5791
begin
58-
body = HTTP::Client.get(URI.parse("https://raw.githubusercontent.com/iv-org/documentation/master/Invidious-Instances.md")).body
92+
# Needs to be replaced once merged!
93+
body = HTTP::Client.get(URI.parse("https://raw.githubusercontent.com/syeopite/documentation/alt-instance-list/instances.yaml")).body
5994
rescue ex
6095
body = ""
6196
end
6297

63-
instances = {} of String => Instance
64-
65-
body = body.split("### Blocked:")[0]
66-
body.scan(/\[(?<host>[^ \]]+)\]\((?<uri>[^\)]+)\)( .(?<region>[\x{1f100}-\x{1f1ff}]{2}))?/mx).each do |md|
67-
region = md["region"]?.try { |region| region.codepoints.map { |codepoint| (codepoint - 0x1f1a5).chr }.join("") }
68-
flag = md["region"]?
69-
70-
uri = URI.parse(md["uri"])
71-
host = md["host"]
72-
73-
case type = host.split(".")[-1]
74-
when "onion"
75-
type = "onion"
76-
77-
if CONFIG["fetch_onion_instance_stats"]?
78-
begin
79-
args = Process.parse_arguments("--socks5-hostname '#{CONFIG["tor_sock_proxy_address"]}:#{CONFIG["tor_sock_proxy_port"]}' 'http://#{uri.host}/api/v1/stats'")
80-
response = nil
81-
Process.run("curl", args: args) do |result|
82-
data = result.output.read_line
83-
response = JSON.parse(data)
84-
end
85-
86-
stats = response
87-
rescue ex
88-
stats = nil
89-
end
90-
end
91-
when "i2p"
92-
else
93-
type = uri.scheme.not_nil!
94-
client = HTTP::Client.new(uri)
95-
client.connect_timeout = 5.seconds
96-
client.read_timeout = 5.seconds
97-
begin
98-
stats = JSON.parse(client.get("/api/v1/stats").body)
99-
rescue ex
100-
stats = nil
101-
end
102-
end
98+
instance_yaml = load_instance_yaml(body)
10399

104-
monitor = monitors.try &.select { |monitor| monitor["name"].try &.as_s == host }[0]?
105-
instances[host] = {flag: flag, region: region, stats: stats, type: type, uri: uri.to_s, monitor: monitor || instances[host]?.try &.[:monitor]?}
106-
end
100+
instance_storage = {} of String => ClearNetInstance | OnionInstance
101+
102+
instance_yaml["instances"]["https"].as_a.each { |i| instance_storage[URI.parse(i["url"].to_s).host.not_nil!] = prepare_http_instance(i, instance_storage, monitors) }
103+
instance_yaml["instances"]["onion"].as_a.each { |i| instance_storage[URI.parse(i["url"].to_s).host.not_nil!] = prepare_onion_instance(i, instance_storage) }
107104

108105
INSTANCES.clear
109-
INSTANCES.merge! instances
106+
INSTANCES.merge! instance_storage
110107

111108
sleep CONFIG["minutes_between_refresh"].as_i.minutes
112109
end
@@ -154,13 +151,13 @@ static_headers do |response, filepath, filestat|
154151
end
155152

156153
SORT_PROCS = {
157-
"health" => ->(name : String, instance : Instance) { -(instance[:monitor]?.try &.["30dRatio"]["ratio"].as_s.to_f || 0.0) },
158-
"location" => ->(name : String, instance : Instance) { instance[:region]? || "ZZ" },
159-
"name" => ->(name : String, instance : Instance) { name },
160-
"signup" => ->(name : String, instance : Instance) { instance[:stats]?.try &.["openRegistrations"]?.try { |bool| bool.as_bool ? 0 : 1 } || 2 },
161-
"type" => ->(name : String, instance : Instance) { instance[:type] },
162-
"users" => ->(name : String, instance : Instance) { -(instance[:stats]?.try &.["usage"]?.try &.["users"]["total"].as_i || 0) },
163-
"version" => ->(name : String, instance : Instance) { instance[:stats]?.try &.["software"]?.try &.["version"].as_s.try &.split("-", 2)[0].split(".").map { |a| -a.to_i } || [0, 0, 0] },
154+
"health" => ->(name : String, instance : ClearNetInstance | OnionInstance) { -(instance[:monitor]?.try &.["30dRatio"]["ratio"].as_s.to_f || 0.0) },
155+
"location" => ->(name : String, instance : ClearNetInstance | OnionInstance) { instance[:region]? || "ZZ" },
156+
"name" => ->(name : String, instance : ClearNetInstance | OnionInstance) { name },
157+
"signup" => ->(name : String, instance : ClearNetInstance | OnionInstance) { instance[:stats]?.try &.["openRegistrations"]?.try { |bool| bool.as_bool ? 0 : 1 } || 2 },
158+
"type" => ->(name : String, instance : ClearNetInstance | OnionInstance) { instance[:type] },
159+
"users" => ->(name : String, instance : ClearNetInstance | OnionInstance) { -(instance[:stats]?.try &.["usage"]?.try &.["users"]["total"].as_i || 0) },
160+
"version" => ->(name : String, instance : ClearNetInstance | OnionInstance) { instance[:stats]?.try &.["software"]?.try &.["version"].as_s.try &.split("-", 2)[0].split(".").map { |a| -a.to_i } || [0, 0, 0] },
164161
}
165162

166163
def sort_instances(instances, sort_by)

src/instances/views/index.ecr

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -118,7 +118,7 @@
118118
<td><%= instance[:type] %></td>
119119
<td><%= instance[:stats]?.try &.["usage"]?.try &.["users"]["total"] || "-" %></td>
120120
<td><%= instance[:stats]?.try &.["openRegistrations"]?.try { |bool| bool.as_bool ? "" : "" } || "-" %></td>
121-
<td><%= instance[:flag]? ? "#{instance[:flag]} #{instance[:region]}" : "-" %></td>
121+
<td><%= instance[:region]? ? "#{instance[:flag]} #{instance[:region]}" : "-" %></td>
122122
<td><%= instance[:monitor]?.try &.["30dRatio"]["ratio"] || "-" %></td>
123123
</tr>
124124
<% end %>

0 commit comments

Comments
 (0)