Update datasources.py (#93)

* Update datasources.py

Removing Hardcoded limit to pdf text.
We kept it for a demo but now is undocumented behavior

* bump version

---------

Co-authored-by: Matthew Russo <mdrusso@mit.edu>

Author: vitaglianog

docs/source/conf.py

@@ -16,7 +16,7 @@
 project = "Palimpzest"
 copyright = "2025, MIT Data Systems Group"
 author = "MIT Data Systems Group"
-release = "0.5.1"
+release = "0.5.2"
 
 # -- General configuration ---------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "palimpzest"
-version = "0.5.1"  # if you update this, be sure to update package version in `docs/source/conf.py` as well
+version = "0.5.2"  # if you update this, be sure to update package version in `docs/source/conf.py` as well
 description = "Palimpzest is a system which enables anyone to process AI-powered analytical queries simply by defining them in a declarative language"
 readme = "README.md"
 requires-python = ">=3.8"

src/palimpzest/core/data/datasources.py

@@ -287,7 +287,7 @@ def get_item(self, idx: int) -> DataRecord:
         dr = DataRecord(self.schema, source_id=filepath)
         dr.filename = pdf_filename
         dr.contents = pdf_bytes
-        dr.text_contents = text_content[:15000]  # TODO Very hacky
+        dr.text_contents = text_content
 
         return dr