@@ -18,6 +18,7 @@ require "http/client"
18
18
require " kemal"
19
19
require " uri"
20
20
21
+ require " ./fetch.cr"
21
22
require " ./helpers/*"
22
23
23
24
CONFIG = load_config()
@@ -28,9 +29,41 @@ macro rendered(filename)
28
29
render " src/instances/views/#{ {{filename}} } .ecr"
29
30
end
30
31
31
- alias Instance = NamedTuple (flag: String ?, region: String ?, stats: JSON ::Any ?, type: String , uri: String , monitor: JSON ::Any ?)
32
-
33
- INSTANCES = {} of String => Instance
32
+ # Nested data within instances
33
+ alias Owner = NamedTuple (name: String , url: String )
34
+ alias Modified = NamedTuple (source: String , changes: String )
35
+ alias Mirrors = NamedTuple (url: String , region: String , flag: String )
36
+
37
+ alias ClearNetInstance = NamedTuple (
38
+ flag: String ,
39
+ region: String ,
40
+ stats: JSON ::Any ?,
41
+ type: String ,
42
+ uri: String ,
43
+ status_url: String ?,
44
+ privacy_policy: String ?,
45
+ ddos_mitm_protection: String ?,
46
+ owner: Owner ,
47
+ modified: Modified ?,
48
+ mirrors: Array (Mirrors )?,
49
+ notes: Array (YAML ::Any )?,
50
+ monitor: JSON ::Any ?)
51
+
52
+ alias OnionInstance = NamedTuple (
53
+ flag: String ,
54
+ region: String ,
55
+ stats: JSON ::Any ?,
56
+ type: String ,
57
+ uri: String ,
58
+ associated_clearnet_instance: String ?,
59
+ privacy_policy: String ?,
60
+ owner: Owner ,
61
+ modified: Modified ?,
62
+ mirrors: Array (Mirrors )?,
63
+ notes: Array (YAML ::Any )?,
64
+ monitor: JSON ::Any ?)
65
+
66
+ INSTANCES = {} of String => ClearNetInstance | OnionInstance
34
67
35
68
spawn do
36
69
loop do
@@ -54,59 +87,23 @@ spawn do
54
87
break
55
88
end
56
89
end
90
+
57
91
begin
58
- body = HTTP ::Client .get(URI .parse(" https://raw.githubusercontent.com/iv-org/documentation/master/Invidious-Instances.md" )).body
92
+ # Needs to be replaced once merged!
93
+ body = HTTP ::Client .get(URI .parse(" https://raw.githubusercontent.com/syeopite/documentation/alt-instance-list/instances.yaml" )).body
59
94
rescue ex
60
95
body = " "
61
96
end
62
97
63
- instances = {} of String => Instance
64
-
65
- body = body.split(" ### Blocked:" )[0 ]
66
- body.scan(/\[ (?<host>[^ \] ] +) \]\( (?<uri>[^\) ] +) \) ( .(?<region>[\x {1f100}-\x {1f1ff}] {2} ) ) ?/mx ).each do |md |
67
- region = md[" region" ]?.try { |region | region.codepoints.map { |codepoint | (codepoint - 0x1f1a5 ).chr }.join(" " ) }
68
- flag = md[" region" ]?
69
-
70
- uri = URI .parse(md[" uri" ])
71
- host = md[" host" ]
72
-
73
- case type = host.split(" ." )[-1 ]
74
- when " onion"
75
- type = " onion"
76
-
77
- if CONFIG [" fetch_onion_instance_stats" ]?
78
- begin
79
- args = Process .parse_arguments(" --socks5-hostname '#{ CONFIG [" tor_sock_proxy_address" ]} :#{ CONFIG [" tor_sock_proxy_port" ]} ' 'http://#{ uri.host } /api/v1/stats'" )
80
- response = nil
81
- Process .run(" curl" , args: args) do |result |
82
- data = result.output.read_line
83
- response = JSON .parse(data)
84
- end
85
-
86
- stats = response
87
- rescue ex
88
- stats = nil
89
- end
90
- end
91
- when " i2p"
92
- else
93
- type = uri.scheme.not_nil!
94
- client = HTTP ::Client .new(uri)
95
- client.connect_timeout = 5 .seconds
96
- client.read_timeout = 5 .seconds
97
- begin
98
- stats = JSON .parse(client.get(" /api/v1/stats" ).body)
99
- rescue ex
100
- stats = nil
101
- end
102
- end
98
+ instance_yaml = load_instance_yaml(body)
103
99
104
- monitor = monitors.try & .select { |monitor | monitor[" name" ].try & .as_s == host }[0 ]?
105
- instances[host] = {flag: flag, region: region, stats: stats, type: type , uri: uri.to_s, monitor: monitor || instances[host]?.try & .[:monitor ]?}
106
- end
100
+ instance_storage = {} of String => ClearNetInstance | OnionInstance
101
+
102
+ instance_yaml[" instances" ][" https" ].as_a.each { |i | instance_storage[URI .parse(i[" url" ].to_s).host.not_nil!] = prepare_http_instance(i, instance_storage, monitors) }
103
+ instance_yaml[" instances" ][" onion" ].as_a.each { |i | instance_storage[URI .parse(i[" url" ].to_s).host.not_nil!] = prepare_onion_instance(i, instance_storage) }
107
104
108
105
INSTANCES .clear
109
- INSTANCES .merge! instances
106
+ INSTANCES .merge! instance_storage
110
107
111
108
sleep CONFIG [" minutes_between_refresh" ].as_i.minutes
112
109
end
@@ -154,13 +151,13 @@ static_headers do |response, filepath, filestat|
154
151
end
155
152
156
153
SORT_PROCS = {
157
- " health" => - > (name : String , instance : Instance ) { - (instance[:monitor ]?.try & .[" 30dRatio" ][" ratio" ].as_s.to_f || 0.0 ) },
158
- " location" => - > (name : String , instance : Instance ) { instance[:region ]? || " ZZ" },
159
- " name" => - > (name : String , instance : Instance ) { name },
160
- " signup" => - > (name : String , instance : Instance ) { instance[:stats ]?.try & .[" openRegistrations" ]?.try { |bool | bool.as_bool ? 0 : 1 } || 2 },
161
- " type" => - > (name : String , instance : Instance ) { instance[:type ] },
162
- " users" => - > (name : String , instance : Instance ) { - (instance[:stats ]?.try & .[" usage" ]?.try & .[" users" ][" total" ].as_i || 0 ) },
163
- " version" => - > (name : String , instance : Instance ) { instance[:stats ]?.try & .[" software" ]?.try & .[" version" ].as_s.try & .split(" -" , 2 )[0 ].split(" ." ).map { |a | - a.to_i } || [0 , 0 , 0 ] },
154
+ " health" => - > (name : String , instance : ClearNetInstance | OnionInstance ) { - (instance[:monitor ]?.try & .[" 30dRatio" ][" ratio" ].as_s.to_f || 0.0 ) },
155
+ " location" => - > (name : String , instance : ClearNetInstance | OnionInstance ) { instance[:region ]? || " ZZ" },
156
+ " name" => - > (name : String , instance : ClearNetInstance | OnionInstance ) { name },
157
+ " signup" => - > (name : String , instance : ClearNetInstance | OnionInstance ) { instance[:stats ]?.try & .[" openRegistrations" ]?.try { |bool | bool.as_bool ? 0 : 1 } || 2 },
158
+ " type" => - > (name : String , instance : ClearNetInstance | OnionInstance ) { instance[:type ] },
159
+ " users" => - > (name : String , instance : ClearNetInstance | OnionInstance ) { - (instance[:stats ]?.try & .[" usage" ]?.try & .[" users" ][" total" ].as_i || 0 ) },
160
+ " version" => - > (name : String , instance : ClearNetInstance | OnionInstance ) { instance[:stats ]?.try & .[" software" ]?.try & .[" version" ].as_s.try & .split(" -" , 2 )[0 ].split(" ." ).map { |a | - a.to_i } || [0 , 0 , 0 ] },
164
161
}
165
162
166
163
def sort_instances (instances, sort_by)
0 commit comments