From 2d09f1d8683ca70991edc6a02241bc043355d2e2 Mon Sep 17 00:00:00 2001 From: pranavsource1 Date: Sun, 22 Mar 2026 02:38:23 +0530 Subject: [PATCH 1/3] LB-1532: Add API endpoints for user data export --- listenbrainz/webserver/__init__.py | 5 +- listenbrainz/webserver/views/export_api.py | 130 ++++++++++++++++++ .../webserver/views/test/test_export_api.py | 70 ++++++++++ 3 files changed, 204 insertions(+), 1 deletion(-) create mode 100644 listenbrainz/webserver/views/export_api.py create mode 100644 listenbrainz/webserver/views/test/test_export_api.py diff --git a/listenbrainz/webserver/__init__.py b/listenbrainz/webserver/__init__.py index f8c07e8a4a..32ea66e6e6 100644 --- a/listenbrainz/webserver/__init__.py +++ b/listenbrainz/webserver/__init__.py @@ -243,7 +243,7 @@ def create_web_app(debug=None): htmx = HTMX(app) # Static files - import listenbrainz.webserver.static_manager + import listenbrainz.webserver.static_manager as static_manager static_manager.read_manifest() app.static_folder = '/static' @@ -457,3 +457,6 @@ def _register_blueprints(app): from listenbrainz.webserver.views.internet_archive_api import internet_archive_api_bp app.register_blueprint(internet_archive_api_bp, url_prefix=API_PREFIX+"/internet_archive") + + from listenbrainz.webserver.views.export_api import export_api_bp + app.register_blueprint(export_api_bp, url_prefix=API_PREFIX+'/export') diff --git a/listenbrainz/webserver/views/export_api.py b/listenbrainz/webserver/views/export_api.py new file mode 100644 index 0000000000..994fb7d3be --- /dev/null +++ b/listenbrainz/webserver/views/export_api.py @@ -0,0 +1,130 @@ +import json +import os +from flask import Blueprint, current_app, jsonify, send_file +from sqlalchemy import text +from psycopg2 import DatabaseError + +from listenbrainz.webserver import db_conn +from listenbrainz.webserver.decorators import api_listenstore_needed +from listenbrainz.webserver.errors import APIBadRequest, APIInternalServerError, APINotFound +from listenbrainz.webserver.views.api_tools import validate_auth_header + +export_api_bp = Blueprint("export_api", __name__) + + +@export_api_bp.route("/", methods=["POST"]) +@api_listenstore_needed +def create_export_task(): + """ Add a request to export the user data to an archive in background. """ + user = validate_auth_header() + user_id = user["id"] + try: + query = """ + INSERT INTO user_data_export (user_id, type, status, progress) + VALUES (:user_id, :type, 'waiting', :progress) + ON CONFLICT (user_id, type) + WHERE status = 'waiting' OR status = 'in_progress' + DO NOTHING + RETURNING id, type, available_until, created, progress, status, filename + """ + result = db_conn.execute(text(query), { + "user_id": user_id, + "type": "export_all_user_data", + "progress": "Your data export will start soon." + }) + export = result.first() + + if export is not None: + query = "INSERT INTO background_tasks (user_id, task, metadata) VALUES (:user_id, :task, :metadata) ON CONFLICT DO NOTHING RETURNING id" + result = db_conn.execute(text(query), { + "user_id": user_id, + "task": "export_all_user_data", + "metadata": json.dumps({"export_id": export.id}) + }) + task = result.first() + if task is not None: + db_conn.commit() + return jsonify({ + "export_id": export.id, + "type": export.type, + "available_until": export.available_until.isoformat() if export.available_until is not None else None, + "created": export.created.isoformat(), + "progress": export.progress, + "status": export.status, + "filename": export.filename, + }) + + # task already exists in queue, rollback new entry + db_conn.rollback() + raise APIBadRequest(message="Data export already requested.") + + except DatabaseError: + current_app.logger.error('Error while exporting user data: %s', user["musicbrainz_id"], exc_info=True) + raise APIInternalServerError(f'Error while exporting user data {user["musicbrainz_id"]}, please try again later.') + + +@export_api_bp.route("/", methods=["GET"]) +@api_listenstore_needed +def get_export_task(export_id): + """ Retrieve the requested export's data if it belongs to the specified user """ + user = validate_auth_header() + user_id = user["id"] + + result = db_conn.execute( + text("SELECT * FROM user_data_export WHERE user_id = :user_id AND id = :export_id"), + {"user_id": user_id, "export_id": export_id} + ) + row = result.first() + if row is None: + raise APINotFound("Export not found") + return jsonify({ + "export_id": row.id, + "type": row.type, + "available_until": row.available_until.isoformat() if row.available_until is not None else None, + "created": row.created.isoformat(), + "progress": row.progress, + "status": row.status, + "filename": row.filename, + }) + + +@export_api_bp.route("/", methods=["GET"]) +@api_listenstore_needed +def list_export_tasks(): + """ Retrieve the all export tasks for the current user """ + user = validate_auth_header() + user_id = user["id"] + + result = db_conn.execute( + text("SELECT * FROM user_data_export WHERE user_id = :user_id ORDER BY created DESC"), + {"user_id": user_id} + ) + rows = result.mappings().all() + return jsonify([{ + "export_id": row["id"], + "type": row["type"], + "available_until": row["available_until"].isoformat() if row["available_until"] is not None else None, + "created": row["created"].isoformat(), + "progress": row["progress"], + "status": row["status"], + "filename": row["filename"], + } for row in rows]) + + +@export_api_bp.route("//download", methods=["GET"]) +@api_listenstore_needed +def download_export_archive(export_id): + """ Download the requested export if it is complete and belongs to the specified user """ + user = validate_auth_header() + user_id = user["id"] + + result = db_conn.execute( + text("SELECT filename FROM user_data_export WHERE user_id = :user_id AND status = 'completed' AND id = :export_id"), + {"user_id": user_id, "export_id": export_id} + ) + row = result.first() + if row is None: + raise APINotFound("Export not found") + + file_path = os.path.join(current_app.config["USER_DATA_EXPORT_BASE_DIR"], str(row["filename"])) + return send_file(file_path, mimetype="application/zip", as_attachment=True) diff --git a/listenbrainz/webserver/views/test/test_export_api.py b/listenbrainz/webserver/views/test/test_export_api.py new file mode 100644 index 0000000000..96dc0f8f10 --- /dev/null +++ b/listenbrainz/webserver/views/test/test_export_api.py @@ -0,0 +1,70 @@ +import unittest +from flask import json +from listenbrainz.tests.integration import ListenAPIIntegrationTestCase +from listenbrainz.db.user import get_or_create + + +class ExportAPIIntegrationTestCase(ListenAPIIntegrationTestCase): + + def setUp(self): + super().setUp() + # Ensure we have a user with a token + self.user = get_or_create(self.db_conn, 1, 'testuser') + self.auth_headers = {'Authorization': f'Token {self.user["auth_token"]}'} + + def test_create_get_list_export(self): + """ Test the complete flow of creating, checking, and listing exports """ + + # 1. Create an export task + response = self.client.post('/1/export/', headers=self.auth_headers) + self.assertEqual(response.status_code, 200) + data = json.loads(response.data) + + self.assertIn('export_id', data) + self.assertEqual(data['status'], 'waiting') + self.assertEqual(data['type'], 'export_all_user_data') + + export_id = data['export_id'] + + # 2. Check the status of the export + response = self.client.get(f'/1/export/{export_id}', headers=self.auth_headers) + self.assertEqual(response.status_code, 200) + data = json.loads(response.data) + + self.assertEqual(data['export_id'], export_id) + # In a test environment without a real background worker, status remains 'waiting' + self.assertEqual(data['status'], 'waiting') + + # 3. List all exports + response = self.client.get('/1/export/', headers=self.auth_headers) + self.assertEqual(response.status_code, 200) + data = json.loads(response.data) + + self.assertIsInstance(data, list) + self.assertGreaterEqual(len(data), 1) + + # Verify our export is in the list + found = False + for export in data: + if export['export_id'] == export_id: + found = True + break + self.assertTrue(found, "Created export not found in list") + + def test_export_unauthorized(self): + """ Test that endpoints require authentication """ + response = self.client.post('/1/export/') + self.assertEqual(response.status_code, 401) + + response = self.client.get('/1/export/') + self.assertEqual(response.status_code, 401) + + def test_get_nonexistent_export(self): + """ Test getting a non-existent export """ + # Assuming 999999 is an ID that doesn't exist + response = self.client.get('/1/export/999999', headers=self.auth_headers) + self.assertEqual(response.status_code, 404) + + +if __name__ == '__main__': + unittest.main() From d12a7dcb01c8f95144735a6c555f4f4f62e2d098 Mon Sep 17 00:00:00 2001 From: pranavsource1 Date: Wed, 22 Apr 2026 22:58:07 +0530 Subject: [PATCH 2/3] took care of the refactoring , can u please check the changes . --- listenbrainz/db/user_data_export.py | 108 +++++++++++++++++++++ listenbrainz/webserver/views/export.py | 96 +++--------------- listenbrainz/webserver/views/export_api.py | 84 +++------------- 3 files changed, 136 insertions(+), 152 deletions(-) create mode 100644 listenbrainz/db/user_data_export.py diff --git a/listenbrainz/db/user_data_export.py b/listenbrainz/db/user_data_export.py new file mode 100644 index 0000000000..c24de7a866 --- /dev/null +++ b/listenbrainz/db/user_data_export.py @@ -0,0 +1,108 @@ +import json +from typing import Optional, Dict, Any, List +from sqlalchemy import text + + +def request_user_data_export(db_conn, user_id: int) -> Optional[Dict[str, Any]]: + """ Add a request to export the user data to an archive in background. """ + query = """ + INSERT INTO user_data_export (user_id, type, status, progress) + VALUES (:user_id, :type, 'waiting', :progress) + ON CONFLICT (user_id, type) + WHERE status = 'waiting' OR status = 'in_progress' + DO NOTHING + RETURNING id, type, available_until, created, progress, status, filename + """ + result = db_conn.execute(text(query), { + "user_id": user_id, + "type": "export_all_user_data", + "progress": "Your data export will start soon." + }) + export = result.first() + + if export is not None: + query = "INSERT INTO background_tasks (user_id, task, metadata) VALUES (:user_id, :task, :metadata) ON CONFLICT DO NOTHING RETURNING id" + result = db_conn.execute(text(query), { + "user_id": user_id, + "task": "export_all_user_data", + "metadata": json.dumps({"export_id": export.id}) + }) + task = result.first() + if task is not None: + return { + "export_id": export.id, + "type": export.type, + "available_until": export.available_until.isoformat() if export.available_until is not None else None, + "created": export.created.isoformat(), + "progress": export.progress, + "status": export.status, + "filename": export.filename, + } + return None + + +def get_export_task(db_conn, user_id: int, export_id: int) -> Optional[Dict[str, Any]]: + """ Retrieve the requested export's data if it belongs to the specified user """ + result = db_conn.execute( + text("SELECT * FROM user_data_export WHERE user_id = :user_id AND id = :export_id"), + {"user_id": user_id, "export_id": export_id} + ) + row = result.first() + if row is None: + return None + + return { + "export_id": row.id, + "type": row.type, + "available_until": row.available_until.isoformat() if row.available_until is not None else None, + "created": row.created.isoformat(), + "progress": row.progress, + "status": row.status, + "filename": row.filename, + } + + +def list_export_tasks(db_conn, user_id: int) -> List[Dict[str, Any]]: + """ Retrieve all export tasks for the current user """ + result = db_conn.execute( + text("SELECT * FROM user_data_export WHERE user_id = :user_id ORDER BY created DESC"), + {"user_id": user_id} + ) + rows = result.mappings().all() + return [{ + "export_id": row["id"], + "type": row["type"], + "available_until": row["available_until"].isoformat() if row["available_until"] is not None else None, + "created": row["created"].isoformat(), + "progress": row["progress"], + "status": row["status"], + "filename": row["filename"], + } for row in rows] + + +def get_completed_export_filename(db_conn, user_id: int, export_id: int) -> Optional[str]: + """ Retrieve the filename of a completed export task """ + result = db_conn.execute( + text("SELECT filename FROM user_data_export WHERE user_id = :user_id AND status = 'completed' AND id = :export_id"), + {"user_id": user_id, "export_id": export_id} + ) + row = result.first() + if row is None: + return None + return str(row.filename) + + +def delete_export_task(db_conn, user_id: int, export_id: int) -> bool: + """ Delete the specified export archive and its background task """ + result = db_conn.execute( + text("DELETE FROM user_data_export WHERE user_id = :user_id AND id = :export_id RETURNING filename"), + {"user_id": user_id, "export_id": export_id} + ) + row = result.first() + if row is not None: + db_conn.execute( + text("DELETE FROM background_tasks WHERE user_id = :user_id AND (metadata->'export_id')::int = :export_id"), + {"user_id": user_id, "export_id": export_id} + ) + return True + return False diff --git a/listenbrainz/webserver/views/export.py b/listenbrainz/webserver/views/export.py index 551d199b01..e6ac834352 100644 --- a/listenbrainz/webserver/views/export.py +++ b/listenbrainz/webserver/views/export.py @@ -4,8 +4,8 @@ from flask import Blueprint, current_app, jsonify, send_file from flask_login import current_user from psycopg2 import DatabaseError -from sqlalchemy import text +from listenbrainz.db import user_data_export from listenbrainz.webserver import db_conn from listenbrainz.webserver.decorators import web_listenstore_needed from listenbrainz.webserver.errors import APIInternalServerError, APINotFound, APIBadRequest @@ -20,40 +20,10 @@ def create_export_task(): """ Add a request to export the user data to an archive in background. """ try: - query = """ - INSERT INTO user_data_export (user_id, type, status, progress) - VALUES (:user_id, :type, 'waiting', :progress) - ON CONFLICT (user_id, type) - WHERE status = 'waiting' OR status = 'in_progress' - DO NOTHING - RETURNING id, type, available_until, created, progress, status, filename - """ - result = db_conn.execute(text(query), { - "user_id": current_user.id, - "type": "export_all_user_data", - "progress": "Your data export will start soon." - }) - export = result.first() - - if export is not None: - query = "INSERT INTO background_tasks (user_id, task, metadata) VALUES (:user_id, :task, :metadata) ON CONFLICT DO NOTHING RETURNING id" - result = db_conn.execute(text(query), { - "user_id": current_user.id, - "task": "export_all_user_data", - "metadata": json.dumps({"export_id": export.id}) - }) - task = result.first() - if task is not None: - db_conn.commit() - return jsonify({ - "export_id": export.id, - "type": export.type, - "available_until": export.available_until.isoformat() if export.available_until is not None else None, - "created": export.created.isoformat(), - "progress": export.progress, - "status": export.status, - "filename": export.filename, - }) + export_data = user_data_export.request_user_data_export(db_conn, current_user.id) + if export_data is not None: + db_conn.commit() + return jsonify(export_data) # task already exists in queue, rollback new entry db_conn.rollback() @@ -69,22 +39,10 @@ def create_export_task(): @web_listenstore_needed def get_export_task(export_id): """ Retrieve the requested export's data if it belongs to the specified user """ - result = db_conn.execute( - text("SELECT * FROM user_data_export WHERE user_id = :user_id AND id = :export_id"), - {"user_id": current_user.id, "export_id": export_id} - ) - row = result.first() - if row is None: + export_data = user_data_export.get_export_task(db_conn, current_user.id, export_id) + if export_data is None: raise APINotFound("Export not found") - return jsonify({ - "export_id": row.id, - "type": row.type, - "available_until": row.available_until.isoformat() if row.available_until is not None else None, - "created": row.created.isoformat(), - "progress": row.progress, - "status": row.status, - "filename": row.filename, - }) + return jsonify(export_data) @export_bp.get("/list/") @@ -92,20 +50,8 @@ def get_export_task(export_id): @web_listenstore_needed def list_export_tasks(): """ Retrieve the all export tasks for the current user """ - result = db_conn.execute( - text("SELECT * FROM user_data_export WHERE user_id = :user_id ORDER BY created DESC"), - {"user_id": current_user.id} - ) - rows = result.mappings().all() - return jsonify([{ - "export_id": row.id, - "type": row.type, - "available_until": row.available_until.isoformat() if row.available_until is not None else None, - "created": row.created.isoformat(), - "progress": row.progress, - "status": row.status, - "filename": row.filename, - } for row in rows]) + export_tasks = user_data_export.list_export_tasks(db_conn, current_user.id) + return jsonify(export_tasks) @export_bp.post("/download//") @@ -113,15 +59,11 @@ def list_export_tasks(): @web_listenstore_needed def download_export_archive(export_id): """ Download the requested export if it is complete and belongs to the specified user """ - result = db_conn.execute( - text("SELECT filename FROM user_data_export WHERE user_id = :user_id AND status = 'completed' AND id = :export_id"), - {"user_id": current_user.id, "export_id": export_id} - ) - row = result.first() - if row is None: + filename = user_data_export.get_completed_export_filename(db_conn, current_user.id, export_id) + if filename is None: raise APINotFound("Export not found") - file_path = os.path.join(current_app.config["USER_DATA_EXPORT_BASE_DIR"], str(row.filename)) + file_path = os.path.join(current_app.config["USER_DATA_EXPORT_BASE_DIR"], filename) return send_file(file_path, mimetype="application/zip", as_attachment=True) @@ -131,16 +73,8 @@ def download_export_archive(export_id): @web_listenstore_needed def delete_export_archive(export_id): """ Delete the specified export archive """ - result = db_conn.execute( - text("DELETE FROM user_data_export WHERE user_id = :user_id AND id = :export_id RETURNING filename"), - {"user_id": current_user.id, "export_id": export_id} - ) - row = result.first() - if row is not None: - db_conn.execute( - text("DELETE FROM background_tasks WHERE user_id = :user_id AND (metadata->'export_id')::int = :export_id"), - {"user_id": current_user.id, "export_id": export_id} - ) + success = user_data_export.delete_export_task(db_conn, current_user.id, export_id) + if success: db_conn.commit() # file is deleted from disk by cronjob return jsonify({"success": True}) diff --git a/listenbrainz/webserver/views/export_api.py b/listenbrainz/webserver/views/export_api.py index 994fb7d3be..99cf042169 100644 --- a/listenbrainz/webserver/views/export_api.py +++ b/listenbrainz/webserver/views/export_api.py @@ -1,9 +1,9 @@ import json import os from flask import Blueprint, current_app, jsonify, send_file -from sqlalchemy import text from psycopg2 import DatabaseError +from listenbrainz.db import user_data_export from listenbrainz.webserver import db_conn from listenbrainz.webserver.decorators import api_listenstore_needed from listenbrainz.webserver.errors import APIBadRequest, APIInternalServerError, APINotFound @@ -19,40 +19,10 @@ def create_export_task(): user = validate_auth_header() user_id = user["id"] try: - query = """ - INSERT INTO user_data_export (user_id, type, status, progress) - VALUES (:user_id, :type, 'waiting', :progress) - ON CONFLICT (user_id, type) - WHERE status = 'waiting' OR status = 'in_progress' - DO NOTHING - RETURNING id, type, available_until, created, progress, status, filename - """ - result = db_conn.execute(text(query), { - "user_id": user_id, - "type": "export_all_user_data", - "progress": "Your data export will start soon." - }) - export = result.first() - - if export is not None: - query = "INSERT INTO background_tasks (user_id, task, metadata) VALUES (:user_id, :task, :metadata) ON CONFLICT DO NOTHING RETURNING id" - result = db_conn.execute(text(query), { - "user_id": user_id, - "task": "export_all_user_data", - "metadata": json.dumps({"export_id": export.id}) - }) - task = result.first() - if task is not None: - db_conn.commit() - return jsonify({ - "export_id": export.id, - "type": export.type, - "available_until": export.available_until.isoformat() if export.available_until is not None else None, - "created": export.created.isoformat(), - "progress": export.progress, - "status": export.status, - "filename": export.filename, - }) + export_data = user_data_export.request_user_data_export(db_conn, user_id) + if export_data is not None: + db_conn.commit() + return jsonify(export_data) # task already exists in queue, rollback new entry db_conn.rollback() @@ -70,22 +40,10 @@ def get_export_task(export_id): user = validate_auth_header() user_id = user["id"] - result = db_conn.execute( - text("SELECT * FROM user_data_export WHERE user_id = :user_id AND id = :export_id"), - {"user_id": user_id, "export_id": export_id} - ) - row = result.first() - if row is None: + export_data = user_data_export.get_export_task(db_conn, user_id, export_id) + if export_data is None: raise APINotFound("Export not found") - return jsonify({ - "export_id": row.id, - "type": row.type, - "available_until": row.available_until.isoformat() if row.available_until is not None else None, - "created": row.created.isoformat(), - "progress": row.progress, - "status": row.status, - "filename": row.filename, - }) + return jsonify(export_data) @export_api_bp.route("/", methods=["GET"]) @@ -95,20 +53,8 @@ def list_export_tasks(): user = validate_auth_header() user_id = user["id"] - result = db_conn.execute( - text("SELECT * FROM user_data_export WHERE user_id = :user_id ORDER BY created DESC"), - {"user_id": user_id} - ) - rows = result.mappings().all() - return jsonify([{ - "export_id": row["id"], - "type": row["type"], - "available_until": row["available_until"].isoformat() if row["available_until"] is not None else None, - "created": row["created"].isoformat(), - "progress": row["progress"], - "status": row["status"], - "filename": row["filename"], - } for row in rows]) + export_tasks = user_data_export.list_export_tasks(db_conn, user_id) + return jsonify(export_tasks) @export_api_bp.route("//download", methods=["GET"]) @@ -118,13 +64,9 @@ def download_export_archive(export_id): user = validate_auth_header() user_id = user["id"] - result = db_conn.execute( - text("SELECT filename FROM user_data_export WHERE user_id = :user_id AND status = 'completed' AND id = :export_id"), - {"user_id": user_id, "export_id": export_id} - ) - row = result.first() - if row is None: + filename = user_data_export.get_completed_export_filename(db_conn, user_id, export_id) + if filename is None: raise APINotFound("Export not found") - file_path = os.path.join(current_app.config["USER_DATA_EXPORT_BASE_DIR"], str(row["filename"])) + file_path = os.path.join(current_app.config["USER_DATA_EXPORT_BASE_DIR"], filename) return send_file(file_path, mimetype="application/zip", as_attachment=True) From b1fa3cf637e8ab9bfc9b79d7d74a4eede2171dee Mon Sep 17 00:00:00 2001 From: pranavsource1 Date: Sat, 25 Apr 2026 02:06:59 +0530 Subject: [PATCH 3/3] resolved test errors --- listenbrainz/webserver/views/export_api.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/listenbrainz/webserver/views/export_api.py b/listenbrainz/webserver/views/export_api.py index 99cf042169..b999e26e5b 100644 --- a/listenbrainz/webserver/views/export_api.py +++ b/listenbrainz/webserver/views/export_api.py @@ -5,7 +5,7 @@ from listenbrainz.db import user_data_export from listenbrainz.webserver import db_conn -from listenbrainz.webserver.decorators import api_listenstore_needed +from listenbrainz.webserver.decorators import api_listenstore_needed, crossdomain from listenbrainz.webserver.errors import APIBadRequest, APIInternalServerError, APINotFound from listenbrainz.webserver.views.api_tools import validate_auth_header @@ -13,6 +13,7 @@ @export_api_bp.route("/", methods=["POST"]) +@crossdomain @api_listenstore_needed def create_export_task(): """ Add a request to export the user data to an archive in background. """ @@ -34,6 +35,7 @@ def create_export_task(): @export_api_bp.route("/", methods=["GET"]) +@crossdomain @api_listenstore_needed def get_export_task(export_id): """ Retrieve the requested export's data if it belongs to the specified user """ @@ -47,6 +49,7 @@ def get_export_task(export_id): @export_api_bp.route("/", methods=["GET"]) +@crossdomain @api_listenstore_needed def list_export_tasks(): """ Retrieve the all export tasks for the current user """ @@ -58,6 +61,7 @@ def list_export_tasks(): @export_api_bp.route("//download", methods=["GET"]) +@crossdomain @api_listenstore_needed def download_export_archive(export_id): """ Download the requested export if it is complete and belongs to the specified user """