Skip to content

Commit 449c025

Browse files
bondianobondiano
committed
fix comment after review
1 parent 8db7d6a commit 449c025

File tree

2 files changed

+57
-34
lines changed

2 files changed

+57
-34
lines changed

‎otus-16/src/otus_16/homework.clj‎

Lines changed: 42 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,12 @@
44
[clojure.string :as str]
55
[clojure.core.reducers :as r]))
66

7+
(def log-regex
8+
#"^([\d+.]+) (\S+) (\S+) (\[[\w+/]+:[\w+:]+ \+\d+\]) \"(.+?)\" (\d{3}) (\d+) \"([^\"]+)\" \"(.+?)\"")
9+
710
(defn parse-log
811
[^String log]
9-
(let [regex #"^([\d+.]+) (\S+) (\S+) (\[[\w+/]+:[\w+:]+ \+\d+\]) \"(.+?)\" (\d{3}) (\d+) \"([^\"]+)\" \"(.+?)\""
10-
[_ ip user-name date-time request response size referer user-agent] (re-find regex log)]
12+
(let [[_ ip _ user-name date-time request response size referer user-agent] (re-find log-regex log)]
1113
{:ip ip
1214
:user-name user-name
1315
:date-time date-time
@@ -19,28 +21,40 @@
1921

2022
(defn get-logs-paths
2123
[dir-path]
22-
(->> (io/as-file dir-path)
23-
(.listFiles)
24-
(filter #(.isFile %))
25-
(map #(.getPath %))))
24+
(->> (io/as-file dir-path)
25+
(file-seq)
26+
(filter #(.isFile %))))
2627

2728
(defn read-logs
2829
[logs]
2930
(map #(io/reader %) logs))
3031

32+
(defn ->url [log]
33+
{:pre [(contains? log :request)]}
34+
(let [request (-> log :request)]
35+
(when request
36+
(-> request
37+
(str/split #" ")
38+
second))))
39+
3140
(defn filter-by-url
3241
[url logs]
3342
(if (= url :all)
3443
logs
35-
(letfn [(->url [log] (-> log :request (str/split #" ") second))]
36-
(r/filter #(= url (->url %)) logs))))
44+
(r/filter #(= url (->url %)) logs)))
45+
46+
(comment
47+
(r/fold str (filter-by-url "test" [{:request "test"} {:request "test2"} {:request "test"}])))
48+
49+
(defn ->referer [log]
50+
{:pre [(contains? log :referer)]}
51+
(-> log :referer))
3752

3853
(defn filter-by-referer
3954
[referer logs]
4055
(if (= referer :all)
4156
logs
42-
(letfn [(->referer [log] (-> log :referer))]
43-
(r/filter #(= referer (->referer %)) logs))))
57+
(r/filter #(= referer (->referer %)) logs)))
4458

4559
(defn close-logs
4660
[logs]
@@ -59,31 +73,26 @@
5973
(r/map #(parse-int-safe (:size %)))
6074
(r/fold +)))
6175

76+
(defn process-partition
77+
[logs {:keys [url referrer]}]
78+
(->> logs
79+
(r/map parse-log)
80+
(filter-by-url url)
81+
(filter-by-referer referrer)))
82+
6283
(defn sum-partition
6384
[partition]
6485
(reduce #(+ %1 (sum-size %2))
6586
0
6687
partition))
6788

6889
(defn process-log
69-
[log-file &
70-
{:keys [url referrer]}]
71-
(->> log-file
72-
(line-seq)
73-
(partition-all 5000)
74-
(pmap #(map parse-log %))
75-
(filter-by-url url)
76-
(filter-by-referer referrer)
77-
(sum-partition)))
78-
79-
(comment
80-
(time
81-
(let [file (io/reader "./logs/access.log.2")]
82-
(-> file
83-
(process-log {:url :all :referrer :all})
84-
(println))
85-
(.close file))))
86-
90+
[log-file filter-params]
91+
(->> log-file
92+
(line-seq)
93+
(partition-all 5000)
94+
(pmap #(process-partition % filter-params))
95+
(sum-partition)))
8796

8897
(defn solution
8998
[& {:keys [url referrer]
@@ -92,14 +101,14 @@
92101
get-logs-paths
93102
read-logs)]
94103
(->> logs
95-
(pmap #(process-log % :url url :referrer referrer))
104+
(pmap #(process-log % {:url url :referrer referrer}))
96105
(reduce +)
97106
(println "Bytes:"))
98107
(close-logs logs)))
99108

100109
(comment
101110
;; возможные вызовы функции
102-
(time (solution))
103-
(solution :url "some-url")
104-
(solution :referrer "some-referrer")
105-
(solution :url "some-url" :referrer "some-referrer"))
111+
(time (solution))
112+
(time (solution :url "/rss/"))
113+
(solution :referrer "some-referrer")
114+
(solution :url "some-url" :referrer "some-referrer"))

‎otus-16/test/otus_16/homework_test.clj‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,20 @@
11
(ns otus-16.homework-test
22
(:require [clojure.test :refer :all]
3-
[otus-16.core :refer :all]))
3+
[otus-16.homework :as sut]))
44

55

6+
(def stub-row "66.249.68.12 - - [20/Sep/2020:22:11:20 +0000] \"GET /%D0%BC%D0%B0%D0%BB%D0%B5%D0%BD%D1%8C%D0%BA%D0%B8%D0%B9-%D0%BC%D0%B0%D0%BB%D1%8C%D1%87%D0%B8%D0%BA-%D0%BF%D0%BE-%D0%B8%D0%BC%D0%B5%D0%BD%D0%B8-%D0%9D%D1%83%D1%80%D0%B1%D0%B5%D0%BA-%D0%B6%D0%B8%D0%BB-%D0%B2-%D0%BD%D0%B5%D0%B1%D0%BE%D0%BB%D1%8C%D1%88%D0%BE%D0%B9/?p=2 HTTP/1.1\" 304 0 \"-\" \"Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.110 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)\" \"-\"")
67

8+
(deftest parse-log-test
9+
(testing "parse-log"
10+
(is (= {:ip "66.249.68.12",
11+
:user-name "-",
12+
:date-time "[20/Sep/2020:22:11:20 +0000]",
13+
:request
14+
"GET /%D0%BC%D0%B0%D0%BB%D0%B5%D0%BD%D1%8C%D0%BA%D0%B8%D0%B9-%D0%BC%D0%B0%D0%BB%D1%8C%D1%87%D0%B8%D0%BA-%D0%BF%D0%BE-%D0%B8%D0%BC%D0%B5%D0%BD%D0%B8-%D0%9D%D1%83%D1%80%D0%B1%D0%B5%D0%BA-%D0%B6%D0%B8%D0%BB-%D0%B2-%D0%BD%D0%B5%D0%B1%D0%BE%D0%BB%D1%8C%D1%88%D0%BE%D0%B9/?p=2 HTTP/1.1",
15+
:response "304",
16+
:size "0",
17+
:referer "-",
18+
:user-agent
19+
"Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.110 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"}
20+
(sut/parse-log stub-row)))))

0 commit comments

Comments
 (0)