Add OpenAI-powered content parsing improvement feature

Author: paskal

backend/extractor/readability.go

@@ -14,6 +14,7 @@ import (
 	"github.com/PuerkitoBio/goquery"
 	log "github.com/go-pkgz/lgr"
 	"github.com/mauidude/go-readability"
+	"github.com/sashabaranov/go-openai"
 	"go.mongodb.org/mongo-driver/bson/primitive"
 
 	"github.com/ukeeper/ukeeper-redabilty/backend/datastore"
@@ -33,6 +34,7 @@ type UReadability struct {
 	TimeOut     time.Duration
 	SnippetSize int
 	Rules       Rules
+	OpenAIKey   string
 }
 
 // Response from api calls
@@ -128,6 +130,113 @@ func (f UReadability) Extract(ctx context.Context, reqURL string) (rb *Response,
 	return rb, nil
 }
 
+// ContentParsedWrong handles the logic for when content is parsed incorrectly
+func (f UReadability) ContentParsedWrong(ctx context.Context, urlStr string) (string, error) {
+	// Extract content using the current method
+	originalContent, err := f.Extract(ctx, urlStr)
+	if err != nil {
+		return "", fmt.Errorf("failed to extract content: %v", err)
+	}
+
+	// Get CSS selector from ChatGPT
+	selector, err := f.getChatGPTSelector(ctx, urlStr)
+	if err != nil {
+		return "", fmt.Errorf("failed to get CSS selector: %v", err)
+	}
+
+	// Get the HTML body
+	body, err := f.getHTMLBody(urlStr)
+	if err != nil {
+		return "", fmt.Errorf("failed to get HTML body: %v", err)
+	}
+
+	// Extract content using the new selector
+	newContent, err := f.extractContentWithSelector(body, selector)
+	if err != nil {
+		return "", fmt.Errorf("failed to extract content with new selector: %v", err)
+	}
+
+	// Compare original and new content
+	if strings.TrimSpace(originalContent.Content) != strings.TrimSpace(newContent) {
+		// Contents are different, create a new rule
+		rule := datastore.Rule{
+			Author:   "",
+			Domain:   f.extractDomain(urlStr),
+			Content:  selector,
+			TestURLs: []string{urlStr},
+			Enabled:  true,
+		}
+
+		_, err = f.Rules.Save(ctx, rule)
+		if err != nil {
+			return "", fmt.Errorf("failed to save new rule: %v", err)
+		}
+
+		return fmt.Sprintf("new custom rule with DOM %s created", selector), nil
+	}
+
+	return "default rule is good, no need to create the custom one", nil
+}
+
+func (f UReadability) getChatGPTSelector(ctx context.Context, urlStr string) (string, error) {
+	client := openai.NewClient(f.OpenAIKey)
+	resp, err := client.CreateChatCompletion(
+		ctx,
+		openai.ChatCompletionRequest{
+			Model: openai.GPT4o,
+			Messages: []openai.ChatCompletionMessage{
+				{
+					Role:    openai.ChatMessageRoleSystem,
+					Content: "You are a helpful assistant that provides CSS selectors for extracting main content from web pages.",
+				},
+				{
+					Role:    openai.ChatMessageRoleUser,
+					Content: fmt.Sprintf("Given the URL %s, identify the CSS selector that can be used to extract the main content of the article. This typically includes elements like 'article', 'main', or specific classes. Return only this selector and nothing else.", urlStr),
+				},
+			},
+		},
+	)
+
+	if err != nil {
+		return "", err
+	}
+
+	return resp.Choices[0].Message.Content, nil
+}
+
+func (f UReadability) getHTMLBody(urlStr string) (string, error) {
+	resp, err := http.Get(urlStr)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return "", err
+	}
+
+	return string(body), nil
+}
+
+func (f UReadability) extractContentWithSelector(body, selector string) (string, error) {
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(body))
+	if err != nil {
+		return "", err
+	}
+
+	content := doc.Find(selector).Text()
+	return content, nil
+}
+
+func (f UReadability) extractDomain(urlStr string) string {
+	u, err := url.Parse(urlStr)
+	if err != nil {
+		return ""
+	}
+	return u.Hostname()
+}
+
 // gets content from raw body string, both content (text only) and rich (with html tags)
 func (f UReadability) getContent(ctx context.Context, body, reqURL string) (content, rich string, err error) {
 	// general parser

backend/main.go

@@ -42,7 +42,7 @@ func main() {
 		log.Fatalf("[ERROR] can't connect to mongo %v", err)
 	}
 	srv := rest.Server{
-		Readability: extractor.UReadability{TimeOut: 30, SnippetSize: 300, Rules: db.GetStores()},
+		Readability: extractor.UReadability{TimeOut: 30, SnippetSize: 300, Rules: db.GetStores(), OpenAIKey: opts.OpenAIKey},
 		Token:       opts.Token,
 		Credentials: opts.Credentials,
 		Version:     revision,

backend/rest/server.go

@@ -83,6 +83,7 @@ func (s *Server) routes(frontendDir string) chi.Router {
 		r.Get("/rule/{id}", s.getRuleByID)
 		r.Get("/rules", s.getAllRules)
 		r.Post("/auth", s.authFake)
+		r.Get("/content-parsed-wrong", s.contentParsedWrong)
 
 		r.Group(func(protected chi.Router) {
 			protected.Use(basicAuth("ureadability", s.Credentials))
@@ -288,6 +289,24 @@ func (s *Server) generateSummary(ctx context.Context, content string) (string, e
 	return resp.Choices[0].Message.Content, nil
 }
 
+func (s *Server) contentParsedWrong(w http.ResponseWriter, r *http.Request) {
+	exampleURL := r.URL.Query().Get("url")
+	if exampleURL == "" {
+		render.Status(r, http.StatusBadRequest)
+		render.JSON(w, r, JSON{"error": "url parameter is required"})
+		return
+	}
+
+	message, err := s.Readability.ContentParsedWrong(r.Context(), exampleURL)
+	if err != nil {
+		render.Status(r, http.StatusBadRequest)
+		render.JSON(w, r, JSON{"error": err.Error()})
+		return
+	}
+
+	render.JSON(w, r, JSON{"message": message})
+}
+
 func getBid(id string) primitive.ObjectID {
 	bid, err := primitive.ObjectIDFromHex(id)
 	if err != nil {

backend/web/index.html

@@ -43,7 +43,12 @@
     </table>
   </div>
 </div>
-
+<div class="content-parsed-wrong">
+  <h2>Report Incorrectly Parsed Content</h2>
+  <input type="text" id="url-input" placeholder="Enter URL">
+  <button id="report-button">Report Incorrect Parsing</button>
+  <div id="result-message"></div>
+</div>
 <div class="footer wrapper page__footer">
   <a href="#" class="link footer__copyright">uReadability</a>,
   <script>document.write((new Date()).getFullYear().toString());</script>

backend/web/main.css

@@ -324,4 +324,26 @@ img {
 
 textarea {
   overflow: auto;
+}
+
+.content-parsed-wrong {
+  margin-top: 20px;
+  padding: 20px;
+  border: 1px solid #ccc;
+  border-radius: 5px;
+}
+
+#url-input {
+  width: 300px;
+  padding: 5px;
+  margin-right: 10px;
+}
+
+#report-button {
+  padding: 5px 10px;
+}
+
+#result-message {
+  margin-top: 10px;
+  font-weight: bold;
 }

backend/web/main.js

@@ -232,6 +232,24 @@ document.addEventListener('DOMContentLoaded', () => {
     if (document.getElementById('rules__list')) {
         loadRules();
     }
+
+    document.getElementById('report-button').addEventListener('click', function () {
+        const url = document.getElementById('url-input').value;
+        if (!url) {
+            alert('Please enter a URL');
+            return;
+        }
+
+        fetch(`/api/content-parsed-wrong?url=${encodeURIComponent(url)}`)
+            .then(response => response.json())
+            .then(data => {
+                document.getElementById('result-message').textContent = data.message;
+            })
+            .catch(error => {
+                console.error('Error:', error);
+                document.getElementById('result-message').textContent = 'An error occurred. Please try again.';
+            });
+    });
 });
 
 function loadRules() {