fix: judge an empty chunk body against its own media type

The previous commit raised on every empty HTTP-200 chunk body, which is
wrong for NDJSON: zero records is zero lines, so an empty body is a
well-formed result and a split whose pages are blank would have failed.

Recombination now takes each chunk's Content-Type. An empty body raises
only when the chunk is not application/x-ndjson -- JSON has no empty
document, so an empty JSON 200 stays a defect, and an unknown media type
is read as JSON, which is what the deployed API returns.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

Author: badGarnet

CHANGELOG.md

@@ -1,7 +1,7 @@
 ## 0.46.1
 
 ### Fixes
-* Fail loudly when a split-PDF chunk returns HTTP 200 with an empty body in NDJSON elements-file mode. The chunk used to be logged and skipped, so the combined `elements_file` was silently short by those pages while the call still returned 200 — and `split_pdf_allow_failed=False` did not catch it, because an empty 200 counts as a successful chunk. Recombination now raises `EmptyChunkResponseError` (a `ValueError`), matching the buffered path, which raises `JSONDecodeError` on the same response. A chunk that legitimately produced no elements returns an empty JSON array and is still accepted.
+* Fail loudly when a split-PDF chunk returns HTTP 200 with an empty JSON body in NDJSON elements-file mode. The chunk used to be logged and skipped, so the combined `elements_file` was silently short by those pages while the call still returned 200 — and `split_pdf_allow_failed=False` did not catch it, because an empty 200 counts as a successful chunk. Recombination now raises `EmptyChunkResponseError` (a `ValueError`), matching the buffered path, which raises `JSONDecodeError` on the same response. Emptiness is judged against the chunk's own `Content-Type`: JSON has no empty document (a chunk with no elements is `[]`), while `application/x-ndjson` encodes zero records as zero lines, so an empty NDJSON chunk is well formed and still contributes nothing.
 
 ## 0.46.0
 

_test_unstructured_client/unit/test_ndjson_elements_file.py

@@ -118,12 +118,16 @@ def test_mixed_chunk_formats(tmp_path):
 
 
 @pytest.mark.parametrize("body", ["", "   ", "\n\n"])
-def test_empty_chunk_files_raise(tmp_path, body):
-    """An empty 200 body must fail, not be skipped.
-
-    Skipping it made the document silently short: this function returns only a path and an
-    element count, so nothing downstream could tell a truncated document from a complete
-    one. `split_pdf_allow_failed` does not cover it either -- the chunk reported success.
+@pytest.mark.parametrize("media_type", [None, "application/json", "application/json; charset=utf-8"])
+def test_empty_json_chunk_files_raise(tmp_path, body, media_type):
+    """An empty 200 body from a JSON chunk must fail, not be skipped.
+
+    JSON has no empty document -- a chunk with no elements is `[]` -- so this is a
+    malformed response. Skipping it made the document silently short: this function returns
+    only a path and an element count, so nothing downstream could tell a truncated document
+    from a complete one. `split_pdf_allow_failed` does not cover it either -- the chunk
+    reported success. An unset `Content-Type` is treated as JSON, which is what the
+    deployed API returns.
     """
     chunk_a = tmp_path / "a.json"
     chunk_empty = tmp_path / "empty.json"
@@ -132,7 +136,18 @@ def test_empty_chunk_files_raise(tmp_path, body):
 
     out = tmp_path / "combined.ndjson"
     with pytest.raises(request_utils.EmptyChunkResponseError, match="empty.json"):
-        combine_chunk_files_to_ndjson([str(chunk_a), str(chunk_empty)], str(out))
+        combine_chunk_files_to_ndjson(
+            [str(chunk_a), str(chunk_empty)], str(out), ["application/json", media_type]
+        )
+
+
+def test_empty_chunk_media_types_default_to_json_when_not_passed(tmp_path):
+    """Callers that cannot supply media types must still get the strict reading."""
+    chunk_empty = tmp_path / "empty.json"
+    chunk_empty.write_text("", encoding="utf-8")
+
+    with pytest.raises(request_utils.EmptyChunkResponseError):
+        combine_chunk_files_to_ndjson([str(chunk_empty)], str(tmp_path / "combined.ndjson"))
 
 
 def test_empty_chunk_error_is_a_value_error(tmp_path):
@@ -141,14 +156,59 @@ def test_empty_chunk_error_is_a_value_error(tmp_path):
     chunk_empty.write_text("", encoding="utf-8")
 
     with pytest.raises(ValueError):
-        combine_chunk_files_to_ndjson([str(chunk_empty)], str(tmp_path / "combined.ndjson"))
+        combine_chunk_files_to_ndjson(
+            [str(chunk_empty)], str(tmp_path / "combined.ndjson"), ["application/json"]
+        )
+
+
+@pytest.mark.parametrize("body", ["", "   ", "\n\n"])
+@pytest.mark.parametrize(
+    "media_type", ["application/x-ndjson", "application/x-ndjson; charset=utf-8", "APPLICATION/X-NDJSON"]
+)
+def test_empty_ndjson_chunk_is_zero_records(tmp_path, body, media_type):
+    """NDJSON spells zero records as zero lines, so an empty body is well formed.
+
+    The counterpart to `test_empty_json_chunk_files_raise`: the same empty file on disk is
+    a defect from a JSON chunk and a legitimate result from an NDJSON one, and the declared
+    media type is the only thing that separates them. Failing here would break a split
+    whose pages are blank.
+    """
+    chunk_a = tmp_path / "a.ndjson"
+    chunk_empty = tmp_path / "empty.ndjson"
+    elements_a = _elements("a", 2)
+    _write_ndjson(chunk_a, elements_a)
+    chunk_empty.write_text(body, encoding="utf-8")
+
+    out = tmp_path / "combined.ndjson"
+    written = combine_chunk_files_to_ndjson(
+        [str(chunk_a), str(chunk_empty)], str(out), ["application/x-ndjson", media_type]
+    )
+
+    assert written == 2
+    assert _read_ndjson(out) == elements_a
+
+
+def test_every_ndjson_chunk_empty_yields_an_empty_output(tmp_path):
+    """A document whose every chunk produced no elements is empty, not an error."""
+    chunk_a = tmp_path / "a.ndjson"
+    chunk_b = tmp_path / "b.ndjson"
+    chunk_a.write_text("", encoding="utf-8")
+    chunk_b.write_text("", encoding="utf-8")
+
+    out = tmp_path / "combined.ndjson"
+    written = combine_chunk_files_to_ndjson(
+        [str(chunk_a), str(chunk_b)], str(out), ["application/x-ndjson"] * 2
+    )
+
+    assert written == 0
+    assert out.read_text(encoding="utf-8") == ""
 
 
 def test_empty_array_chunk_contributes_nothing(tmp_path):
     """A chunk that legitimately produced no elements (e.g. blank pages).
 
-    The counterpart to `test_empty_chunk_files_raise`: `[]` and an empty body are
-    different responses on the wire, and only the latter is a defect.
+    The JSON counterpart to `test_empty_ndjson_chunk_is_zero_records`: `[]` and an empty
+    body are different responses on the wire, and only the latter is a defect.
     """
     chunk_a = tmp_path / "a.json"
     chunk_b = tmp_path / "b.json"
@@ -396,16 +456,31 @@ def test_server_cannot_name_a_local_file_via_a_response_header(tmp_path):
 # --- hook-level temp-file lifecycle ------------------------------------------------
 
 
-def _ndjson_response(elements):
+def _ndjson_response(elements, media_type="application/x-ndjson"):
     body = "".join(json.dumps(e) + "\n" for e in elements).encode()
-    return httpx.Response(status_code=200, content=body)
+    headers = {"content-type": media_type} if media_type else {}
+    return httpx.Response(status_code=200, headers=headers, content=body)
+
+
+def _cached_chunk_response(path, media_type="application/x-ndjson"):
+    """What the cached path leaves behind: the body replaced by its temp-file path.
+
+    Mirrors `_await_elements`' cached branch, which streams the body to disk and rebuilds
+    the response with `content=temp_file_name` and the server's original headers -- so the
+    declared media type still describes the file's contents, not the path in the body.
+    """
+    return httpx.Response(
+        status_code=200,
+        headers={"content-type": media_type} if media_type else {},
+        content=str(path).encode(),
+    )
 
 
-def _hook_in_ndjson_mode(operation_id, tmp_path):
+def _hook_in_ndjson_mode(operation_id, tmp_path, cache_tmp_data=False):
     """A hook set up as `before_request` would leave it for an uncached NDJSON run."""
     hook = SplitPdfHook()
     hook.ndjson_mode[operation_id] = True
-    hook.cache_tmp_data_feature[operation_id] = False
+    hook.cache_tmp_data_feature[operation_id] = cache_tmp_data
     hook.cache_tmp_data_dir[operation_id] = str(tmp_path)
     hook.allow_failed[operation_id] = False
     # Marks the operation live; `_clear_operation` removing it is what signals teardown.
@@ -562,8 +637,9 @@ def test_malformed_chunk_leaves_no_partial_output_behind(tmp_path):
     assert list(Path(tmp_path).glob("*.partial")) == []
 
 
-def test_empty_chunk_response_fails_the_operation(tmp_path):
-    """An empty 200 chunk must fail the whole partition, as the buffered path does.
+@pytest.mark.parametrize("media_type", ["application/json", None])
+def test_empty_json_chunk_response_fails_the_operation(tmp_path, media_type):
+    """An empty 200 JSON chunk must fail the whole partition, as the buffered path does.
 
     Driven through the hook because that is where the consequence lives: previously the
     chunk was skipped and `_build_after_success_response` handed back a combined file that
@@ -572,9 +648,10 @@ def test_empty_chunk_response_fails_the_operation(tmp_path):
     """
     operation_id = "op-empty-chunk"
     hook = _hook_in_ndjson_mode(operation_id, tmp_path)
+    headers = {"content-type": media_type} if media_type else {}
     responses = [
         (0, _ndjson_response(_elements("a", 2))),
-        (1, httpx.Response(status_code=200, content=b"")),
+        (1, httpx.Response(status_code=200, headers=headers, content=b"")),
     ]
 
     with pytest.raises(request_utils.EmptyChunkResponseError):
@@ -585,6 +662,51 @@ def test_empty_chunk_response_fails_the_operation(tmp_path):
     assert list(Path(tmp_path).glob("*.partial")) == []
 
 
+def test_empty_ndjson_chunk_response_is_accepted_in_memory_mode(tmp_path):
+    """A zero-record NDJSON chunk is a valid result and must not fail the partition.
+
+    The default path (`cache_tmp_data` off): the empty body is spilled to disk, so the
+    media type carried by the response is the only thing that distinguishes it from the
+    malformed JSON case above.
+    """
+    operation_id = "op-empty-ndjson-memory"
+    hook = _hook_in_ndjson_mode(operation_id, tmp_path)
+    elements_a, elements_c = _elements("a", 2), _elements("c", 3)
+    responses = [
+        (0, _ndjson_response(elements_a)),
+        (1, _ndjson_response([])),  # a chunk of blank pages
+        (2, _ndjson_response(elements_c)),
+    ]
+
+    hook._elements_from_task_responses(operation_id, responses, started_at=0.0)
+
+    combined = hook.ndjson_output_path[operation_id]
+    assert _read_ndjson(combined) == elements_a + elements_c
+    # The spilled chunk files are gone; only the combined output survives.
+    assert _ndjson_files_in(tmp_path) == [Path(combined).name]
+
+
+def test_empty_ndjson_chunk_response_is_accepted_in_cached_mode(tmp_path):
+    """Cached counterpart: the chunk file on disk is empty and the body is its path."""
+    operation_id = "op-empty-ndjson-cached"
+    hook = _hook_in_ndjson_mode(operation_id, tmp_path, cache_tmp_data=True)
+    # Cached chunk files live in the operation's tempdir, kept out of the directory the
+    # combined file lands in so the assertion below stays unambiguous.
+    cache_dir = tmp_path / "cache"
+    cache_dir.mkdir()
+    elements_a = _elements("a", 2)
+    chunk_a, chunk_empty = cache_dir / "a.json", cache_dir / "empty.json"
+    _write_ndjson(chunk_a, elements_a)
+    chunk_empty.write_text("", encoding="utf-8")
+    responses = [(0, _cached_chunk_response(chunk_a)), (1, _cached_chunk_response(chunk_empty))]
+
+    hook._elements_from_task_responses(operation_id, responses, started_at=0.0)
+
+    combined = hook.ndjson_output_path[operation_id]
+    assert _read_ndjson(combined) == elements_a
+    assert _ndjson_files_in(tmp_path) == [Path(combined).name]
+
+
 def test_no_combined_file_is_created_when_every_chunk_failed(tmp_path):
     operation_id = "op-all-failed"
     hook = _hook_in_ndjson_mode(operation_id, tmp_path)

src/unstructured_client/_hooks/custom/request_utils.py

@@ -291,18 +291,27 @@ def create_response(elements: list) -> httpx.Response:
 
 
 class EmptyChunkResponseError(ValueError):
-    """A split-PDF chunk returned HTTP 200 with a body that holds no elements at all.
+    """A split-PDF chunk returned HTTP 200 with a body its own media type cannot produce.
 
-    An empty body is not the same wire response as an empty JSON array: `[]` is a chunk
-    that legitimately produced no elements (blank pages) and is accepted, while nothing
-    at all is a defect. Skipping it would make the document silently short, and
-    `split_pdf_allow_failed` cannot catch it because the chunk reports success.
+    Whether an empty body is a defect depends on the format the chunk claims. JSON has no
+    empty document -- a chunk with no elements is `[]` -- so nothing at all is malformed,
+    and skipping it would make the document silently short with no way for the caller to
+    notice (`split_pdf_allow_failed` does not cover it, because the chunk reports
+    success). NDJSON is a record per line, so an empty body is a well-formed zero-record
+    response and is accepted.
 
     Subclasses `ValueError` so that it lands where the buffered path's `JSONDecodeError`
     (also a `ValueError`) already does for callers that guard the parse.
     """
 
 
+def _is_ndjson_media_type(media_type: Optional[str]) -> bool:
+    """Whether a chunk's `Content-Type` declares NDJSON, ignoring any parameters."""
+    if not media_type:
+        return False
+    return media_type.split(";", 1)[0].strip().lower() == NDJSON_MEDIA_TYPE
+
+
 def _first_non_space_char(stream: TextIO) -> str:
     """Return the first non-whitespace character in `stream`, or "" if there is none."""
     while True:
@@ -314,7 +323,11 @@ def _first_non_space_char(stream: TextIO) -> str:
             return stripped[0]
 
 
-def combine_chunk_files_to_ndjson(chunk_paths: list[str], out_path: str) -> int:
+def combine_chunk_files_to_ndjson(
+    chunk_paths: list[str],
+    out_path: str,
+    media_types: Optional[list[Optional[str]]] = None,
+) -> int:
     """Combine per-chunk split-PDF response files into one NDJSON file on disk.
 
     Recombining chunks by parsing them builds four full copies of the document (a list
@@ -329,35 +342,55 @@ def combine_chunk_files_to_ndjson(chunk_paths: list[str], out_path: str) -> int:
 
     NDJSON chunks are copied through without parsing.
 
+    An empty body is read against the chunk's declared media type, which is the only
+    thing that distinguishes the two cases: NDJSON encodes zero records as zero lines, so
+    an empty NDJSON chunk contributes nothing and is fine; JSON has no empty document, so
+    an empty one is malformed and raises rather than silently shortening the result.
+    A chunk whose media type is unknown is treated as JSON -- that is what the deployed
+    API returns, and guessing NDJSON would reinstate the silent truncation.
+
     Args:
         chunk_paths: Per-chunk response files, in element order.
         out_path: File to write the combined NDJSON to.
+        media_types: The `Content-Type` each chunk was served with, positionally matching
+            `chunk_paths`. Omit when the media types are not known.
 
     Returns:
         The number of elements written.
 
     Raises:
-        EmptyChunkResponseError: A chunk returned 200 with an empty body.
+        EmptyChunkResponseError: A non-NDJSON chunk returned 200 with an empty body.
     """
     total = 0
     with open(out_path, "w", encoding="utf-8") as out:
-        for chunk_path in chunk_paths:
+        for index, chunk_path in enumerate(chunk_paths):
+            media_type = media_types[index] if media_types is not None else None
             with open(chunk_path, "r", encoding="utf-8") as chunk:
                 first_char = _first_non_space_char(chunk)
                 if not first_char:
-                    # A 200 with an empty body. Fail loudly: skipping it makes the document
-                    # silently short with no way for the caller to notice, since this
-                    # function's only output is the combined path. The buffered path raises
-                    # `JSONDecodeError` on the same response, so NDJSON mode must not turn
-                    # a hard failure into truncation.
+                    if _is_ndjson_media_type(media_type):
+                        # Zero records, spelled the only way NDJSON can spell it.
+                        logger.debug(
+                            "split_pdf event=ndjson_empty_chunk file=%s media_type=%s",
+                            os.path.basename(chunk_path),
+                            media_type,
+                        )
+                        continue
+                    # A JSON 200 with an empty body. Fail loudly: skipping it makes the
+                    # document silently short with no way for the caller to notice, since
+                    # this function's only output is the combined path. The buffered path
+                    # raises `JSONDecodeError` on the same response, so NDJSON mode must
+                    # not turn a hard failure into truncation.
                     logger.error(
-                        "split_pdf event=ndjson_empty_chunk file=%s",
+                        "split_pdf event=empty_chunk_body file=%s media_type=%s",
                         os.path.basename(chunk_path),
+                        media_type,
                     )
                     raise EmptyChunkResponseError(
                         "A split-PDF chunk returned HTTP 200 with an empty body "
-                        f"({os.path.basename(chunk_path)}); its elements would be missing "
-                        "from the combined output. An empty result must be an empty JSON "
+                        f"({os.path.basename(chunk_path)}, Content-Type: "
+                        f"{media_type or 'unset'}); its elements would be missing from the "
+                        "combined output. A JSON chunk with no elements must be an empty "
                         "array."
                     )
                 chunk.seek(0)