refs # scrawler

scrawler/main.go

@@ -0,0 +1,72 @@
+package main
+
+import (
+	"encoding/json"
+	"flag"
+	"log"
+	"os"
+	"strconv"
+	"strings"
+
+	"github.com/gocolly/colly"
+)
+
+type comment struct {
+	Author  string `selector:"a.hnuser"`
+	URL     string `selector:".age a[href]" attr:"href"`
+	Comment string `selector:".comment"`
+	Replies []*comment
+	depth   int
+}
+
+func main() {
+	var itemID string
+	flag.StringVar(&itemID, "id", "", "hackernews post id")
+	flag.Parse()
+
+	if itemID == "" {
+		log.Println("Hackernews post id required")
+		os.Exit(1)
+	}
+
+	comments := make([]*comment, 0)
+
+	// Instantiate default collector
+	c := colly.NewCollector()
+
+	// Extract comment
+	c.OnHTML(".comment-tree tr.athing", func(e *colly.HTMLElement) {
+		width, err := strconv.Atoi(e.ChildAttr("td.ind img", "width"))
+		if err != nil {
+			return
+		}
+		// hackernews uses 40px spacers to indent comment replies,
+		// so we have to divide the width with it to get the depth
+		// of the comment
+		depth := width / 40
+		c := &comment{
+			Replies: make([]*comment, 0),
+			depth:   depth,
+		}
+		e.Unmarshal(c)
+		c.Comment = strings.TrimSpace(c.Comment[:len(c.Comment)-5])
+		if depth == 0 {
+			comments = append(comments, c)
+			return
+		}
+		parent := comments[len(comments)-1]
+		// append comment to its parent
+		for i := 0; i < depth-1; i++ {
+			parent = parent.Replies[len(parent.Replies)-1]
+		}
+		parent.Replies = append(parent.Replies, c)
+	})
+
+	c.Visit("https://news.ycombinator.com/item?id=" + itemID)
+
+	enc := json.NewEncoder(os.Stdout)
+	enc.SetIndent("", "  ")
+
+	// Dump json to the standard output
+	enc.Encode(comments)
+}