Added crawler for osu

Author: MewX

WebCrawlers/osu.ppy.sh/README.md

@@ -0,0 +1,2 @@
+This crawler is for crawling the music packs from osu,
+and they will be used for preparing a mobile game project!

WebCrawlers/osu.ppy.sh/getlists.py

@@ -0,0 +1,56 @@
+# this file is used for getting pack lists and store them in the database
+import re
+import socket
+import urllib.request as urllib2
+
+# the list can be get from: https://osu.ppy.sh/p/packlist?t=s
+import settings
+from property_db import PropertyDb
+
+FULL_LIST = 'https://osu.ppy.sh/p/packlist?t=s'
+LIST_PATTERN = r'"expandPack\(\'(.+?)\'\)"'
+MAP_URL_PATTERN = 'https://osu.ppy.sh/pages/include/packlist-info.php?n={}'
+PAGE_PATTERN = r'<h2>(.+?)</h2>.+?<div>.+?href="(.+?)">'  # group 1 - file name; group 2 - link
+socket.setdefaulttimeout(15)
+
+"""
+download a page by passed url
+"""
+def download(url):
+    while True:
+        try:
+            opener = urllib2.build_opener()
+            opener.addheaders.append(('Cookie', settings.COOKIES))
+            opener.addheaders.append(('User-Agent', settings.USER_AGENT))
+            f = opener.open(url)
+            b = f.read()
+        except Exception as e:
+            print(e)
+            print('-- trying again: {}'.format(url))
+            continue
+        return b
+
+
+"""
+the main function
+"""
+list_page = download(FULL_LIST)
+propertyDb = PropertyDb()
+for match in re.finditer(LIST_PATTERN, list_page, re.DOTALL | re.MULTILINE):
+    key = match.group(1)
+    print('working on pack: {}'.format(key))
+
+    # if this pack is not downloaded
+    if propertyDb.does_record_exist(key):
+        continue
+
+    current_page = download(MAP_URL_PATTERN.format(key))
+    page_matcher = re.search(PAGE_PATTERN, current_page, re.DOTALL | re.MULTILINE)
+
+    if page_matcher:
+        propertyDb.save_or_overwrite_data(key, current_page)
+    else:
+        # if not found
+        print('-- ERROR not found things on page on key {}: {}'.format(key, current_page))
+
+print('done')

WebCrawlers/osu.ppy.sh/property_db.py

@@ -0,0 +1,29 @@
+# the file for handling the property db
+import sqlite3
+
+import settings
+
+"""
+the property-like db
+"""
+class PropertyDb:
+
+    def __init__(self):
+        self.conn = sqlite3.connect(settings.DB_NAME)
+        self.init_db()
+
+
+    def init_db(self):
+        self.conn.execute('CREATE TABLE IF NOT EXISTS properties (name TEXT PRIMARY KEY, value TEXT, time DATETIME DEFAULT current_timestamp);')
+
+
+    def does_record_exist(self, key):
+        return self.get_value(key) is not None
+
+    def save_or_overwrite_data(self, key, value):
+        self.conn.execute("INSERT OR REPLACE INTO properties(name, value, time) VALUES (?, ?, current_timestamp);", (key, value,))
+
+
+    def get_value(self, key):
+        self.conn.execute("SELECT * FROM properties WHERE name = ?;", (key,))
+        return self.conn.fetchone()

WebCrawlers/osu.ppy.sh/settings.py

@@ -0,0 +1,4 @@
+# NOTE: DO NOT COMMIT THIS FILE AFTER FILLING SECRET INFORMATION
+DB_NAME = 'url.db'
+COOKIES = ''  # TODO: fill this
+USER_AGENT = 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Safari/537.36'