feat: do not generate overwritten file output for CSVs

Author: PaulKalho

main.py

@@ -2,7 +2,7 @@
 import pandas as pd
 
 from pathlib import Path
-from typing import List
+from typing import Callable, List, Optional
 from scystream.sdk.core import entrypoint
 from scystream.sdk.env.settings import (
     EnvSettings,
@@ -108,7 +108,10 @@ class PreprocessCSV(EnvSettings):
 
     csv_input: CSVFileInput
     normalized_docs_output: NormalizedDocsOutput
-    normalized_overwritten_file_output: NormalizedCSVOutput
+
+
+#    TODO: As CSVs might be very large, we do not write the csv output
+#    normalized_overwritten_file_output: NormalizedCSVOutput
 
 
 def _write_preprocessed_docs_to_postgres(
@@ -135,7 +138,7 @@ def _write_preprocessed_docs_to_postgres(
 
 def _preprocess_and_store(
     documents: List[DocumentRecord],
-    overwrite_callback,
+    overwrite_callback: Optional[Callable],
     settings,
 ) -> List[PreprocessedDocument]:
 
@@ -158,14 +161,15 @@ def _preprocess_and_store(
     )
 
     # Overwrite file using injected behavior
-    export_path = Path(
-        f"output.{settings.normalized_overwritten_file_output.FILE_EXT}"
-    )
-    overwrite_callback(result, export_path)
+    if overwrite_callback:
+        export_path = Path(
+            f"output.{settings.normalized_overwritten_file_output.FILE_EXT}"
+        )
+        overwrite_callback(result, export_path)
 
-    S3Operations.upload(
-        settings.normalized_overwritten_file_output, export_path
-    )
+        S3Operations.upload(
+            settings.normalized_overwritten_file_output, export_path
+        )
 
     logger.info("Preprocessing completed successfully.")
     return result
@@ -215,6 +219,6 @@ def preprocess_csv_file(settings):
 
     _preprocess_and_store(
         documents=loader.document_records,
-        overwrite_callback=loader.overwrite_with_results,
+        overwrite_callback=None,
         settings=settings,
     )

output.bib

@@ -0,0 +1,79 @@
+@article{WOS:001016714700004,
+ abstract = {modern contemporari transhuman seen recent rise academ popular relev specif naiv metaphys idea immort return rise modern contemporari contemporari transhuman transhuman seen seen recent recent rise rise academ academ popular popular relev relev specif specif naiv naiv metaphys metaphys idea idea immort immort return return rise modern contemporari transhuman contemporari transhuman seen transhuman seen recent seen recent rise recent rise academ rise academ popular academ popular relev popular relev specif relev specif naiv specif naiv metaphys naiv metaphys idea metaphys idea immort idea immort return immort return rise articl refrain ethic polit assess transhuman articl refrain refrain ethic ethic polit polit assess assess transhuman articl refrain ethic refrain ethic polit ethic polit assess polit assess transhuman critiqu exact metaphys idealist natur transhuman pursuit digit immort idea technolog advanc precis artifici gener intellig immort virtual self possibl critiqu exact exact metaphys metaphys idealist idealist natur natur transhuman transhuman pursuit pursuit digit digit immort immort idea idea technolog technolog advanc advanc precis precis artifici artifici gener gener intellig intellig immort immort virtual virtual self self possibl critiqu exact metaphys exact metaphys idealist metaphys idealist natur idealist natur transhuman natur transhuman pursuit transhuman pursuit digit pursuit digit immort digit immort idea immort idea technolog idea technolog advanc technolog advanc precis advanc precis artifici precis artifici gener artifici gener intellig gener intellig immort intellig immort virtual immort virtual self virtual self possibl articl follow form immanuel kant paralog critiqu pure reason kant concern substanti immort natur soul experienti imposs articl follow follow form form immanuel immanuel kant kant paralog paralog critiqu critiqu pure pure reason reason kant kant concern concern substanti substanti immort immort natur natur soul soul experienti experienti imposs articl follow form follow form immanuel form immanuel kant immanuel kant paralog kant paralog critiqu paralog critiqu pure critiqu pure reason pure reason kant reason kant concern kant concern substanti concern substanti immort substanti immort natur immort natur soul natur soul experienti soul experienti imposs articl offer theoret practic paralog fals logic infer argu transhumanist claim digit immort possibl fundament stem incorrect major premis articl offer offer theoret theoret practic practic paralog paralog fals fals logic logic infer infer argu argu transhumanist transhumanist claim claim digit digit immort immort possibl possibl fundament fundament stem stem incorrect incorrect major major premis articl offer theoret offer theoret practic theoret practic paralog practic paralog fals paralog fals logic fals logic infer logic infer argu infer argu transhumanist argu transhumanist claim transhumanist claim digit claim digit immort digit immort possibl immort possibl fundament possibl fundament stem fundament stem incorrect stem incorrect major incorrect major premis concern substanti natur inform inform theoret paralog second concern infinit transform pure plastic inform practic paralog concern substanti substanti natur natur inform inform inform inform theoret theoret paralog paralog second second concern concern infinit infinit transform transform pure pure plastic plastic inform inform practic practic paralog concern substanti natur substanti natur inform natur inform inform inform inform theoret inform theoret paralog theoret paralog second paralog second concern second concern infinit concern infinit transform infinit transform pure transform pure plastic pure plastic inform plastic inform practic inform practic paralog},
+ address = {2-4 PARK SQUARE, MILTON PARK, ABINGDON OX14 4RN, OXON, ENGLAND},
+ author = {White, Joel},
+ author-email = {jhmw01@gmail.com},
+ da = {2025-06-26},
+ doc-delivery-number = {K5GF0},
+ doi = {10.1080/20539320.2022.2150463},
+ eissn = {2053-9339},
+ issn = {2053-9320},
+ journal = {JOURNAL OF AESTHETICS AND PHENOMENOLOGY},
+ journal-iso = {J. Aesthet. Phenomenol.},
+ keywords = {Transhumanism; Critical Philosophy; Immanuel Kant; Entropy; Paralogisms;
+Digital Immortality},
+ language = {English},
+ month = {JUL 3},
+ number = {2, SI},
+ number-of-cited-references = {30},
+ orcid-numbers = {White, Joel/0000-0001-6460-0564},
+ pages = {155-172},
+ publisher = {ROUTLEDGE JOURNALS, TAYLOR \& FRANCIS LTD},
+ research-areas = {Philosophy},
+ times-cited = {0},
+ title = {Theoretical and Practical Paralogisms of Digital Immortality},
+ type = {Article},
+ unique-id = {WOS:001016714700004},
+ usage-count-last-180-days = {3},
+ usage-count-since-2013 = {15},
+ volume = {9},
+ web-of-science-categories = {Philosophy},
+ web-of-science-index = {Emerging Sources Citation Index (ESCI)},
+ year = {2022}
+}
+
+@article{WOS:001322577100012,
+ abstract = {unit nation panel digit cooper emphas inclus growth digit network digit public good util multistakehold system approach unit nation nation panel panel digit digit cooper cooper emphas emphas inclus inclus growth growth digit digit network network digit digit public public good good util util multistakehold multistakehold system system approach unit nation panel nation panel digit panel digit cooper digit cooper emphas cooper emphas inclus emphas inclus growth inclus growth digit growth digit network digit network digit network digit public digit public good public good util good util multistakehold util multistakehold system multistakehold system approach similarli inform commun technolog ICT innov intervent program govern india digit north east vision emphas need inclus growth ICT northeast region similarli inform inform commun commun technolog technolog ICT ICT innov innov intervent intervent program program govern govern india india digit digit north north east east vision vision emphas emphas need need inclus inclus growth growth ICT ICT northeast northeast region similarli inform commun inform commun technolog commun technolog ICT technolog ICT innov ICT innov intervent innov intervent program intervent program govern program govern india govern india digit india digit north digit north east north east vision east vision emphas vision emphas need emphas need inclus need inclus growth inclus growth ICT growth ICT northeast ICT northeast region line articl present insight field studi conduct rural part manipur india incident found applic rural part develop world line articl articl present present insight insight field field studi studi conduct conduct rural rural part part manipur manipur india india incident incident found found applic applic rural rural part part develop develop world line articl present articl present insight present insight field insight field studi field studi conduct studi conduct rural conduct rural part rural part manipur part manipur india manipur india incident india incident found incident found applic found applic rural applic rural part rural part develop part develop world articl envis commun driven sociodigit transform northeast region india articl envis envis commun commun driven driven sociodigit sociodigit transform transform northeast northeast region region india articl envis commun envis commun driven commun driven sociodigit driven sociodigit transform sociodigit transform northeast transform northeast region northeast region india quest articl highlight sociopolit challeng digit transform provid insight inclus ICT region infrastructur util citizen smart govern servic demand digit empower citizen social welfar capac build commun engag quest articl articl highlight highlight sociopolit sociopolit challeng challeng digit digit transform transform provid provid insight insight inclus inclus ICT ICT region region infrastructur infrastructur util util citizen citizen smart smart govern govern servic servic demand demand digit digit empower empower citizen citizen social social welfar welfar capac capac build build commun commun engag quest articl highlight articl highlight sociopolit highlight sociopolit challeng sociopolit challeng digit challeng digit transform digit transform provid transform provid insight provid insight inclus insight inclus ICT inclus ICT region ICT region infrastructur region infrastructur util infrastructur util citizen util citizen smart citizen smart govern smart govern servic govern servic demand servic demand digit demand digit empower digit empower citizen empower citizen social citizen social welfar social welfar capac welfar capac build capac build commun build commun engag},
+ address = {10662 LOS VAQUEROS CIRCLE, PO BOX 3014, LOS ALAMITOS, CA 90720-1314 USA},
+ affiliation = {Kant, V (Corresponding Author), Indian Inst Technol Kanpur, Kanpur 208016, India.
+Kant, Vivek, Indian Inst Technol Kanpur, Kanpur 208016, India.
+Khanganba, Sanjram Premjit, Indian Inst Technol Indore, Indore 452020, India.
+Dixit, Sudhir, Basic Internet Fdn, Oslo, Norway.},
+ affiliations = {Indian Institute of Technology System (IIT System); Indian Institute of
+Technology (IIT) - Kanpur; Indian Institute of Technology System (IIT
+System); Indian Institute of Technology (IIT) - Indore},
+ author = {Kant, Vivek and Khanganba, Sanjram Premjit and Dixit, Sudhir},
+ author-email = {vkant@iitk.ac.in
+sanjrampk@iiti.ac.in
+sudhir.dixit@ieee.org},
+ da = {2025-06-26},
+ doc-delivery-number = {H3O9D},
+ doi = {10.1109/MITP.2024.3433459},
+ eissn = {1941-045X},
+ issn = {1520-9202},
+ journal = {IT PROFESSIONAL},
+ journal-iso = {IT Prof.},
+ keywords = {Technological innovation; Digital transformation; Government; Buildings;
+Asia; Africa; Information and communication technology},
+ language = {English},
+ month = {JUL-AUG},
+ number = {4},
+ number-of-cited-references = {7},
+ orcid-numbers = {/0000-0002-6215-7500},
+ pages = {42-47},
+ publisher = {IEEE COMPUTER SOC},
+ research-areas = {Computer Science; Telecommunications},
+ researcherid-numbers = {/ITU-6308-2023},
+ times-cited = {0},
+ title = {Sociopolitical Challenges to Digital Transformation of Rural
+Communities: Learnings from a Case Study From Manipur, India},
+ type = {Article},
+ unique-id = {WOS:001322577100012},
+ usage-count-last-180-days = {11},
+ usage-count-since-2013 = {22},
+ volume = {26},
+ web-of-science-categories = {Computer Science, Information Systems; Computer Science, Software
+Engineering; Telecommunications},
+ web-of-science-index = {Science Citation Index Expanded (SCI-EXPANDED)},
+ year = {2024}
+}

output.txt

@@ -0,0 +1,4 @@
+cat chase mice cat chase chase mice cat chase mice dog chase cat dog chase chase cat dog chase cat
+bird fli high bird fli fli high bird fli high cat dog coexist cat dog dog coexist cat dog coexist
+mice hide cat mice hide hide cat mice hide cat bird sing loudli bird sing sing loudli bird sing loudli
+cat dog coexist cat dog dog coexist cat dog coexist cat dog coexist cat dog dog coexist cat dog coexist

test/test_full.py

@@ -211,14 +211,6 @@ def test_full_csv(s3_minio):
         ),
         "normalized_docs_DB_TABLE": "normalized_docs_csv",
         "normalized_docs_DB_SCHEMA": DB_SCHEMA,
-        # overwritten CSV output
-        "normalized_overwritten_file_output_S3_HOST": "http://127.0.0.1",
-        "normalized_overwritten_file_output_S3_PORT": "9000",
-        "normalized_overwritten_file_output_S3_ACCESS_KEY": MINIO_USER,
-        "normalized_overwritten_file_output_S3_SECRET_KEY": MINIO_PWD,
-        "normalized_overwritten_file_output_BUCKET_NAME": BUCKET_NAME,
-        "normalized_overwritten_file_output_FILE_PATH": "",
-        "normalized_overwritten_file_output_FILE_NAME": OUTPUT_FILE_NAME,
     }
 
     for k, v in env.items():