Skip to content

Rewrite parser for iv-org/documentation#74 #30

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 7 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,12 @@ COPY ./src/ ./src/
RUN crystal build ./src/instances.cr --release

FROM alpine:latest
RUN apk add --no-cache gc pcre libgcc
RUN apk add --no-cache gc pcre libgcc yaml
WORKDIR /app
RUN addgroup -g 1000 -S invidious && \
adduser -u 1000 -S invidious -G invidious
COPY ./assets/ ./assets/
COPY ./config.yml ./config.yml
COPY --from=builder /app/instances .

EXPOSE 3000
Expand Down
9 changes: 9 additions & 0 deletions config.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
# Uses the system's CURL binary to do so.
fetch_onion_instance_stats: true

# TOR's Sock proxy address that CURL uses to connect to hidden services
tor_sock_proxy_address: "127.0.0.1"
tor_sock_proxy_port: 9050

# Minutes before refreshing the instance stats
minutes_between_refresh: 30
8 changes: 8 additions & 0 deletions docker-compose.yml
Original file line number Diff line number Diff line change
@@ -1,5 +1,13 @@
version: '3'
services:
tor-socks-proxy:
container_name: tor-socks-proxy
image: peterdavehello/tor-socks-proxy:latest
ports:
- "127.0.0.1:8853:53/udp"
- "127.0.0.1:9150:9150/tcp"
restart: unless-stopped

instances:
build: .
restart: unless-stopped
Expand Down
117 changes: 117 additions & 0 deletions src/fetch.cr
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
def fetch_country(md)
region = md["flag"]?.try { |region| region.codepoints.map { |codepoint| (codepoint - 0x1f1a5).chr }.join("") }
flag = md["flag"]?
country_name = md["country_name"]?

return flag, region, country_name
end

def fetch_notes(md)
notes = md["notes"].strip("|")
if notes.empty?
notes = nil
end

return notes
end

def prepare_http_instance(md, instances, monitors)
uri = URI.parse(md["uri"])
host = md["host"]

flag, region, country_name = fetch_country(md)

status_url = md["status_url"]?

privacy_policy = md["privacy_policy"]?

ddos_protection = md["ddos_protection"].strip
if ddos_protection == "None"
ddos_protection = nil
end

owner = {name: md["owner"].strip("@"), url: md["owner_url"]}
notes = fetch_notes(md)

client = HTTP::Client.new(uri)
client.connect_timeout = 5.seconds
client.read_timeout = 5.seconds

begin
stats = JSON.parse(client.get("/api/v1/stats").body)
rescue ex
stats = nil
end

monitor = monitors.try &.select { |monitor| monitor["name"].try &.as_s == host }[0]?
return {flag: flag, region: region, country_name: country_name, stats: stats, type: "https", uri: uri.to_s, status_url: status_url,
privacy_policy: privacy_policy, ddos_protection: ddos_protection,
owner: owner, notes: notes, monitor: monitor || instances[host]?.try &.[:monitor]?}
end

def prepare_onion_instance(md, instances)
uri = URI.parse(md["uri"])
host = md["host"]

clearnet_url = md["clearnet_url"]
flag, region, country_name = fetch_country(md)
privacy_policy = md["privacy_policy"]?
owner = {name: md["owner"].strip("@"), url: md["owner_url"]}
notes = fetch_notes(md)

if CONFIG["fetch_onion_instance_stats"]?
begin
args = Process.parse_arguments("--socks5-hostname '#{CONFIG["tor_sock_proxy_address"]}:#{CONFIG["tor_sock_proxy_port"]}' 'http://#{uri.host}/api/v1/stats'")
response = nil
Process.run("curl", args: args) do |result|
data = result.output.read_line
response = JSON.parse(data)
end

stats = response
rescue ex
stats = nil
end
else
stats = nil
end

return {flag: flag, region: region, country_name: country_name, stats: stats, type: "onion", uri: uri.to_s, clearnet_url: clearnet_url,
privacy_policy: privacy_policy, owner: owner, notes: notes,
monitor: nil}
end

def get_clearnet_instances(body, instances, monitors)
# Crystal currently lacks a markdown parser that supports tables. So...
clear_net_regexes = [
/\[(?<host>[^ \]]+)\]\((?<uri>[^\)]+)\)/, # Address column
/(?<flag>[\x{1f100}-\x{1f1ff}]{2}) (?<country_name>[^ ]+)/, # Country column
/((\[[^\]]+\]\(.*\){1}\])\((?<status_url>.*)\)|(None))/, # Status column
/((\[[^ \]]+\]\((?<privacy_policy>[^\)]+)\))|(None))/, # privacy policy column
/(?<ddos_protection>.*)/, # DDOS protection column
/\[(?<owner>[^ \]]+)\]\((?<owner_url>[^\)]+)\)/, # Owner column
/(?<notes>.*)/, # Note column
]

body.scan(/#{clear_net_regexes.join(/ +\| +/)}/mx).each do |md|
host = md["host"]
instances[host] = prepare_http_instance(md, instances, monitors)
end
end

def get_onion_instances(body, instances)
# Crystal currently lacks a markdown parser that supports tables. So...
clear_net_regexes = [
/\[(?<host>[^ \]]+)\]\((?<uri>[^\)]+)\)/, # Address column
/(?<flag>[\x{1f100}-\x{1f1ff}]{2}) (?<country_name>[^ ]+)/, # Country column
/\[(?<clearnet_host>[^ \]]+)\]\((?<clearnet_url>[^\)]+)\)/, # Clearnet instance column
/((\[[^ \]]+\]\((?<privacy_policy>[^\)]+)\))|(None))/, # privacy policy column
/\[(?<owner>[^ \]]+)\]\((?<owner_url>[^\)]+)\)/, # Owner column
/(?<notes>.*)/, # Notes column
]

body.scan(/#{clear_net_regexes.join(/ +\| +/)}/mx).each do |md|
host = md["host"]
instances[host] = prepare_onion_instance(md, instances)
end
end
6 changes: 6 additions & 0 deletions src/helpers/helpers.cr
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
require "yaml"

def load_config
config = YAML.parse(File.read("config.yml"))
return config
end
61 changes: 22 additions & 39 deletions src/instances.cr
Original file line number Diff line number Diff line change
Expand Up @@ -18,15 +18,22 @@ require "http/client"
require "kemal"
require "uri"

require "./fetch.cr"
require "./helpers/helpers.cr"

CONFIG = load_config()

Kemal::CLI.new ARGV

macro rendered(filename)
render "src/instances/views/#{{{filename}}}.ecr"
end

alias Instance = NamedTuple(flag: String?, region: String?, stats: JSON::Any?, type: String, uri: String, monitor: JSON::Any?)
alias Owner = NamedTuple(name: String, url: String)
alias ClearNetInstance = NamedTuple(flag: String?, region: String?, country_name: String?, stats: JSON::Any?, type: String, uri: String, status_url: String?, privacy_policy: String?, ddos_protection: String?, owner: Owner, notes: String?, monitor: JSON::Any?)
alias OnionInstance = NamedTuple(flag: String?, region: String?, country_name: String?, stats: JSON::Any?, type: String, uri: String, clearnet_url: String?, privacy_policy: String?, owner: Owner, notes: String?, monitor: JSON::Any?)

INSTANCES = {} of String => Instance
INSTANCES = {} of String => ClearNetInstance | OnionInstance

spawn do
loop do
Expand All @@ -51,44 +58,20 @@ spawn do
end
end
begin
body = HTTP::Client.get(URI.parse("https://raw.githubusercontent.com/iv-org/documentation/master/Invidious-Instances.md")).body
# Needs to be replaced once merged!
body = HTTP::Client.get(URI.parse("https://raw.githubusercontent.com/TheFrenchGhosty/documentation/instances-list-rewrite/Public-Instances.md")).body
rescue ex
body = ""
end

instances = {} of String => Instance

body = body.split("### Blocked:")[0]
body.scan(/\[(?<host>[^ \]]+)\]\((?<uri>[^\)]+)\)( .(?<region>[\x{1f100}-\x{1f1ff}]{2}))?/mx).each do |md|
region = md["region"]?.try { |region| region.codepoints.map { |codepoint| (codepoint - 0x1f1a5).chr }.join("") }
flag = md["region"]?

uri = URI.parse(md["uri"])
host = md["host"]

case type = host.split(".")[-1]
when "onion"
when "i2p"
else
type = uri.scheme.not_nil!
client = HTTP::Client.new(uri)
client.connect_timeout = 5.seconds
client.read_timeout = 5.seconds
begin
stats = JSON.parse(client.get("/api/v1/stats").body)
rescue ex
stats = nil
end
end

monitor = monitors.try &.select { |monitor| monitor["name"].try &.as_s == host }[0]?
instances[host] = {flag: flag, region: region, stats: stats, type: type, uri: uri.to_s, monitor: monitor || instances[host]?.try &.[:monitor]?}
end
instances = {} of String => ClearNetInstance | OnionInstance
get_clearnet_instances(body, instances, monitors)
get_onion_instances(body, instances)

INSTANCES.clear
INSTANCES.merge! instances

sleep 5.minutes
sleep CONFIG["minutes_between_refresh"].as_i.minutes
end
end

Expand Down Expand Up @@ -134,13 +117,13 @@ static_headers do |response, filepath, filestat|
end

SORT_PROCS = {
"health" => ->(name : String, instance : Instance) { -(instance[:monitor]?.try &.["30dRatio"]["ratio"].as_s.to_f || 0.0) },
"location" => ->(name : String, instance : Instance) { instance[:region]? || "ZZ" },
"name" => ->(name : String, instance : Instance) { name },
"signup" => ->(name : String, instance : Instance) { instance[:stats]?.try &.["openRegistrations"]?.try { |bool| bool.as_bool ? 0 : 1 } || 2 },
"type" => ->(name : String, instance : Instance) { instance[:type] },
"users" => ->(name : String, instance : Instance) { -(instance[:stats]?.try &.["usage"]?.try &.["users"]["total"].as_i || 0) },
"version" => ->(name : String, instance : Instance) { instance[:stats]?.try &.["software"]?.try &.["version"].as_s.try &.split("-", 2)[0].split(".").map { |a| -a.to_i } || [0, 0, 0] },
"health" => ->(name : String, instance : ClearNetInstance | OnionInstance) { -(instance[:monitor]?.try &.["30dRatio"]["ratio"].as_s.to_f || 0.0) },
"location" => ->(name : String, instance : ClearNetInstance | OnionInstance) { instance[:region]? || "ZZ" },
"name" => ->(name : String, instance : ClearNetInstance | OnionInstance) { name },
"signup" => ->(name : String, instance : ClearNetInstance | OnionInstance) { instance[:stats]?.try &.["openRegistrations"]?.try { |bool| bool.as_bool ? 0 : 1 } || 2 },
"type" => ->(name : String, instance : ClearNetInstance | OnionInstance) { instance[:type] },
"users" => ->(name : String, instance : ClearNetInstance | OnionInstance) { -(instance[:stats]?.try &.["usage"]?.try &.["users"]["total"].as_i || 0) },
"version" => ->(name : String, instance : ClearNetInstance | OnionInstance) { instance[:stats]?.try &.["software"]?.try &.["version"].as_s.try &.split("-", 2)[0].split(".").map { |a| -a.to_i } || [0, 0, 0] },
}

def sort_instances(instances, sort_by)
Expand Down