Merge pull request #96 from rasulkireev/cursor/analyze-project-sitemap-pages-daily-4d41

Analyze project sitemap pages daily

Author: rasulkireev

CHANGELOG.md

@@ -14,6 +14,20 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 
+## [0.0.7] - 2025-11-01
+### Changed
+- update to user-settings page
+- Enhanced blog post generation to intelligently use project pages based on `always_use` flag
+
+### Added
+- sitemaps support
+- project pages in the ui for projects
+- add the ability to select which project pages will always be used in project generations.
+- blog posts use project pages more intelligently with two-tier system:
+  - Required pages (always_use=True) must be linked in generated content
+  - Optional pages are suggested for AI to use intelligently based on relevance
+
+
 ## [0.0.7] - 2025-10-29
 ## Changed
 - Fixed and improved all limitations based on plans.

core/admin.py

@@ -7,6 +7,7 @@
     GeneratedBlogPost,
     Profile,
     Project,
+    ProjectPage,
 )
 
 admin.site.register(Profile)
@@ -15,3 +16,4 @@
 admin.site.register(BlogPostTitleSuggestion)
 admin.site.register(GeneratedBlogPost)
 admin.site.register(AutoSubmissionSetting)
+admin.site.register(ProjectPage)

core/agents.py

@@ -3,6 +3,7 @@
 from core.schemas import (
     GeneratedBlogPostSchema,
     ProjectDetails,
+    ProjectPageDetails,
     TitleSuggestion,
     WebPageContent,
 )
@@ -53,3 +54,29 @@ def add_webpage_content(ctx: RunContext[WebPageContent]) -> str:
         f"Description: {ctx.deps.description}"
         f"Content: {ctx.deps.markdown_content}"
     )
+
+
+########################################################
+
+summarize_page_agent = Agent(
+    "google-gla:gemini-2.5-flash",
+    output_type=ProjectPageDetails,
+    deps_type=WebPageContent,
+    system_prompt=(
+        "You are an expert content summarizer. Based on the web page content provided, "
+        "create a concise 2-3 sentence summary that captures the main purpose and key "
+        "information of the page. Focus on what the page is about and its main value proposition."
+    ),
+    retries=2,
+    model_settings={"temperature": 0.5},
+)
+
+
+@summarize_page_agent.system_prompt
+def add_page_content(ctx: RunContext[WebPageContent]) -> str:
+    return (
+        "Web page content to summarize:"
+        f"Title: {ctx.deps.title}"
+        f"Description: {ctx.deps.description}"
+        f"Content: {ctx.deps.markdown_content}"
+    )

core/api/schemas.py

@@ -12,6 +12,7 @@ class ProfileSettingsOut(Schema):
 class ProjectSettingsOut(Schema):
     name: str
     url: str
+    sitemap_url: str
     has_auto_submission_setting: bool
 
 
@@ -227,3 +228,27 @@ class GetKeywordDetailsOut(Schema):
     status: str
     message: str | None = None
     keyword: KeywordMetricsOut | None = None
+
+
+class UpdateSitemapUrlIn(Schema):
+    project_id: int
+    sitemap_url: str
+
+
+class SubmitSitemapIn(Schema):
+    sitemap_url: str
+
+
+class UpdateSitemapUrlOut(Schema):
+    status: str
+    message: str
+
+
+class ToggleProjectPageAlwaysUseIn(Schema):
+    page_id: int
+
+
+class ToggleProjectPageAlwaysUseOut(Schema):
+    status: str
+    always_use: bool
+    message: str | None = None

core/api/views.py

@@ -3,6 +3,7 @@
 from django.shortcuts import get_object_or_404
 from django.template.loader import render_to_string
 from django.utils import timezone
+from django_q.tasks import async_task
 from ninja import NinjaAPI
 
 from core.api.auth import session_auth, superuser_api_auth
@@ -28,10 +29,15 @@
     ProjectScanIn,
     ProjectScanOut,
     SubmitFeedbackIn,
+    SubmitSitemapIn,
     ToggleAutoSubmissionOut,
     ToggleProjectKeywordUseIn,
     ToggleProjectKeywordUseOut,
+    ToggleProjectPageAlwaysUseIn,
+    ToggleProjectPageAlwaysUseOut,
     UpdateArchiveStatusIn,
+    UpdateSitemapUrlIn,
+    UpdateSitemapUrlOut,
     UpdateTitleScoreIn,
     UserSettingsOut,
     ValidateUrlIn,
@@ -406,6 +412,92 @@ def toggle_auto_submission(request: HttpRequest, project_id: int):
     return {"status": "success", "enabled": project.enable_automatic_post_submission}
 
 
+@api.post("/projects/update-sitemap-url", response=UpdateSitemapUrlOut, auth=[session_auth])
+def update_sitemap_url(request: HttpRequest, data: UpdateSitemapUrlIn):
+    """
+    Update the sitemap URL for a project. When a sitemap URL is added or updated,
+    it triggers automatic parsing and analysis of the sitemap pages.
+    """
+    profile = request.auth
+    project = get_object_or_404(Project, id=data.project_id, profile=profile)
+
+    sitemap_url = data.sitemap_url.strip()
+
+    if not sitemap_url:
+        return {
+            "status": "error",
+            "message": "Sitemap URL cannot be empty",
+        }
+
+    if not sitemap_url.startswith(("http://", "https://")):
+        return {
+            "status": "error",
+            "message": "Sitemap URL must start with http:// or https://",
+        }
+
+    logger.info(
+        "[Update Sitemap URL] Updating sitemap URL for project",
+        project_id=project.id,
+        profile_id=profile.id,
+        sitemap_url=sitemap_url,
+    )
+
+    project.sitemap_url = sitemap_url
+    project.save(update_fields=["sitemap_url"])
+
+    # Trigger sitemap parsing task
+    async_task("core.tasks.parse_sitemap_and_save_urls", project.id, group="Parse Sitemap")
+
+    return {
+        "status": "success",
+        "message": "Sitemap URL updated successfully. Pages will be analyzed in batches of 10.",
+    }
+
+
+@api.post(
+    "/project/{project_id}/sitemap/submit/", response=UpdateSitemapUrlOut, auth=[session_auth]
+)
+def submit_sitemap(request: HttpRequest, project_id: int, data: SubmitSitemapIn):
+    """
+    Submit/update the sitemap URL for a project. When a sitemap URL is added or updated,
+    it triggers automatic parsing and analysis of the sitemap pages.
+    """  # noqa: E501
+    profile = request.auth
+    project = get_object_or_404(Project, id=project_id, profile=profile)
+
+    sitemap_url = data.sitemap_url.strip()
+
+    if not sitemap_url:
+        return {
+            "status": "error",
+            "message": "Sitemap URL cannot be empty",
+        }
+
+    if not sitemap_url.startswith(("http://", "https://")):
+        return {
+            "status": "error",
+            "message": "Sitemap URL must start with http:// or https://",
+        }
+
+    logger.info(
+        "[Submit Sitemap] Submitting sitemap URL for project",
+        project_id=project.id,
+        profile_id=profile.id,
+        sitemap_url=sitemap_url,
+    )
+
+    project.sitemap_url = sitemap_url
+    project.save(update_fields=["sitemap_url"])
+
+    # Trigger sitemap parsing task
+    async_task("core.tasks.parse_sitemap_and_save_urls", project.id, group="Parse Sitemap")
+
+    return {
+        "status": "success",
+        "message": "Sitemap submitted successfully! Your pages will be analyzed shortly.",
+    }
+
+
 @api.post("/update-title-score/{suggestion_id}", response={200: dict}, auth=[session_auth])
 def update_title_score(request: HttpRequest, suggestion_id: int, data: UpdateTitleScoreIn):
     profile = request.auth
@@ -554,6 +646,7 @@ def user_settings(request: HttpRequest, project_id: int):
         project_data = {
             "name": project.name,
             "url": project.url,
+            "sitemap_url": project.sitemap_url,
             "has_auto_submission_setting": project.has_auto_submission_setting,
         }
         data = {"profile": profile_data, "project": project_data}
@@ -780,7 +873,14 @@ def submit_blog_post(request: HttpRequest, data: BlogPostIn):
         )
         return BlogPostOut(status="success", message="Blog post submitted successfully.")
     except Exception as e:
-        return BlogPostOut(status="error", message=f"Failed to submit blog post: {str(e)}")
+        logger.error(
+            "[Submit Blog Post] Failed to submit blog post",
+            error=str(e),
+            exc_info=True,
+            title=data.title,
+            slug=data.slug,
+        )
+        return BlogPostOut(status="error", message="Failed to submit blog post")
 
 
 @api.post("/post-generated-blog-post", response=PostGeneratedBlogPostOut, auth=[session_auth])
@@ -846,3 +946,39 @@ def fix_generated_blog_post(request: HttpRequest, data: FixGeneratedBlogPostIn):
             exc_info=True,
         )
         return {"status": "error", "message": f"Failed to fix blog post: {str(e)}"}
+
+
+@api.post(
+    "/project-pages/toggle-always-use", response=ToggleProjectPageAlwaysUseOut, auth=[session_auth]
+)
+def toggle_project_page_always_use(request: HttpRequest, data: ToggleProjectPageAlwaysUseIn):
+    """
+    Toggle the always_use field for a ProjectPage.
+    When enabled, the page link will always be included in generated blog posts.
+    """  # noqa: E501
+    profile = request.auth
+
+    try:
+        project_page = get_object_or_404(ProjectPage, id=data.page_id, project__profile=profile)
+
+        project_page.always_use = not project_page.always_use
+        project_page.save(update_fields=["always_use"])
+
+        return {
+            "status": "success",
+            "always_use": project_page.always_use,
+        }
+
+    except Exception as error:
+        logger.error(
+            "Failed to toggle ProjectPage always_use field",
+            error=str(error),
+            exc_info=True,
+            page_id=data.page_id,
+            profile_id=profile.id,
+        )
+        return {
+            "status": "error",
+            "always_use": False,
+            "message": f"Failed to toggle always use: {str(error)}",
+        }

core/choices.py

@@ -101,3 +101,8 @@ class KeywordDataSource(models.TextChoices):
 class BlogPostStatus(models.TextChoices):
     DRAFT = "DRAFT"
     PUBLISHED = "PUBLISHED"
+
+
+class ProjectPageSource(models.TextChoices):
+    AI = "AI", "AI"
+    SITEMAP = "SITEMAP", "Sitemap"

core/migrations/0033_project_sitemap_url_projectpage_source.py

@@ -0,0 +1,23 @@
+# Generated by Django 5.2.7 on 2025-11-01 11:10
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0032_add_starts_with_header_validation'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='project',
+            name='sitemap_url',
+            field=models.URLField(blank=True, default='', max_length=500),
+        ),
+        migrations.AddField(
+            model_name='projectpage',
+            name='source',
+            field=models.CharField(choices=[('AI', 'AI'), ('SITEMAP', 'Sitemap')], default='AI', help_text='Source of the page: AI-discovered or from Sitemap', max_length=20),
+        ),
+    ]

core/migrations/0034_alter_projectpage_unique_together_and_more.py

@@ -0,0 +1,22 @@
+# Generated by Django 5.2.7 on 2025-11-01 12:06
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0033_project_sitemap_url_projectpage_source'),
+    ]
+
+    operations = [
+        migrations.AlterUniqueTogether(
+            name='projectpage',
+            unique_together={('project', 'url')},
+        ),
+        migrations.AddField(
+            model_name='projectpage',
+            name='always_use',
+            field=models.BooleanField(default=False, help_text='When enabled, this page link will always be included in generated blog posts'),
+        ),
+    ]

core/model_utils.py

@@ -126,9 +126,9 @@ def get_markdown_content(url):
 
     except requests.exceptions.RequestException as e:
         logger.error(
-            "Error fetching content from Jina Reader",
+            "[Get Markdown Content] Error fetching content from Jina Reader",
             error=str(e),
             exc_info=True,
             url=url,
         )
-        raise e
+        return ("", "", "")