Merge pull request #6262 from thc202/spider/max-size-zero

spider: handle zero as unlimited parse size

Author: psiinon

addOns/spider/CHANGELOG.md

@@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## Unreleased
 ### Changed
 - Maintenance changes.
+- Handle zero max parse size as unlimited.
 
 ## [0.13.0] - 2025-01-10
 ### Changed

addOns/spider/src/main/java/org/zaproxy/addon/spider/SpiderParam.java

@@ -355,7 +355,8 @@ protected void parseImpl() {
 
         this.acceptCookies = getBoolean(SPIDER_ACCEPT_COOKIES, true);
 
-        this.maxParseSizeBytes = getInt(SPIDER_MAX_PARSE_SIZE_BYTES, DEFAULT_MAX_PARSE_SIZE_BYTES);
+        this.maxParseSizeBytes =
+                Math.max(0, getInt(SPIDER_MAX_PARSE_SIZE_BYTES, DEFAULT_MAX_PARSE_SIZE_BYTES));
 
         loadIrrelevantParameters();
         this.confirmRemoveIrrelevantParameter =
@@ -1021,14 +1022,14 @@ public boolean isAcceptCookies() {
     /**
      * Sets the maximum size, in bytes, that a response might have to be parsed.
      *
-     * <p>This allows the spider to skip big responses/files.
+     * <p>This allows the spider to skip big responses/files. 0 for no limit.
      *
      * @param maxParseSizeBytes the maximum size, in bytes, that a response might have to be parsed.
      * @see #getMaxParseSizeBytes()
      */
     public void setMaxParseSizeBytes(int maxParseSizeBytes) {
-        this.maxParseSizeBytes = maxParseSizeBytes;
-        getConfig().setProperty(SPIDER_MAX_PARSE_SIZE_BYTES, maxParseSizeBytes);
+        this.maxParseSizeBytes = Math.max(0, maxParseSizeBytes);
+        getConfig().setProperty(SPIDER_MAX_PARSE_SIZE_BYTES, this.maxParseSizeBytes);
     }
 
     /**

addOns/spider/src/main/java/org/zaproxy/addon/spider/filters/DefaultParseFilter.java

@@ -117,7 +117,8 @@ public FilterResult filtered(HttpMessage responseMessage) {
         }
 
         // Check response body size
-        if (responseMessage.getResponseBody().length() > params.getMaxParseSizeBytes()) {
+        if (params.getMaxParseSizeBytes() != 0
+                && responseMessage.getResponseBody().length() > params.getMaxParseSizeBytes()) {
             getLogger()
                     .debug("Resource too large: {}", responseMessage.getRequestHeader().getURI());
             return filterResultMaxSize;

addOns/spider/src/main/javahelp/org/zaproxy/addon/spider/resources/help/contents/options.html

@@ -43,7 +43,7 @@ <h3>Maximum children to crawl</h3>
 
 	<h3>Maximum parse size</h3>
 	Defines the maximum size, in bytes, that a response might have to be parsed. This allows
-	the Spider to skip big responses/files.
+	the Spider to skip big responses/files. Zero means unlimited size.
 
 	<h3>Domains Always in Scope</h3>
 	Allows to manage the domains, string literals or regular expressions, that are in the

addOns/spider/src/main/resources/org/zaproxy/addon/spider/resources/Messages.properties

@@ -54,7 +54,7 @@ spider.api.action.setOptionMaxDepth.param.Integer =
 spider.api.action.setOptionMaxDuration = 
 spider.api.action.setOptionMaxDuration.param.Integer = 
 spider.api.action.setOptionMaxParseSizeBytes = Sets the maximum size, in bytes, that a response might have to be parsed. This allows the spider to skip big responses/files.
-spider.api.action.setOptionMaxParseSizeBytes.param.Integer = 
+spider.api.action.setOptionMaxParseSizeBytes.param.Integer = The maximum size or 0 for unlimited.
 spider.api.action.setOptionMaxScansInUI = 
 spider.api.action.setOptionMaxScansInUI.param.Integer = 
 spider.api.action.setOptionParseComments = 
@@ -102,7 +102,7 @@ spider.api.view.optionHandleParameters =
 spider.api.view.optionMaxChildren = Gets the maximum number of child nodes (per node) that can be crawled, 0 means no limit.
 spider.api.view.optionMaxDepth = Gets the maximum depth the spider can crawl, 0 if unlimited.
 spider.api.view.optionMaxDuration = 
-spider.api.view.optionMaxParseSizeBytes = Gets the maximum size, in bytes, that a response might have to be parsed.
+spider.api.view.optionMaxParseSizeBytes = Gets the maximum size, in bytes, that a response might have to be parsed, or 0 for unlimited.
 spider.api.view.optionMaxScansInUI = 
 spider.api.view.optionParseComments = 
 spider.api.view.optionParseGit = 
@@ -172,7 +172,7 @@ spider.custom.label.handleOdata = Handle OData Parameters:
 spider.custom.label.maxChildren = Maximum Children to Crawl (0 is unlimited):
 spider.custom.label.maxDepth = Maximum Depth to Crawl (0 is unlimited):
 spider.custom.label.maxDuration = Maximum Duration (minutes; 0 is unlimited):
-spider.custom.label.maxParseSizeBytes = Maximum Parse Size (bytes):
+spider.custom.label.maxParseSizeBytes = Maximum Parse Size (bytes; 0 is unlimited):
 spider.custom.label.parseComments = Parse HTML Comments:
 spider.custom.label.parseDsStore = Parse .DS_Store Files:
 spider.custom.label.parseGit = Parse Git Metadata:
@@ -245,7 +245,7 @@ spider.options.label.handlehodataparameters = Handle OData-specific parameters
 spider.options.label.handleparameters = Query parameters handling for checking visited URIs:
 spider.options.label.irrelevantparameters = Irrelevant parameters:
 spider.options.label.maxChildren = Maximum Children to Crawl (0 is unlimited):
-spider.options.label.maxParseSizeBytes = Maximum Parse Size (bytes):
+spider.options.label.maxParseSizeBytes = Maximum Parse Size (bytes; 0 is unlimited):
 spider.options.label.post = POST forms (recommended but may generate unwanted requests)
 spider.options.label.processform = Process forms (forms are processed and GET queries submitted)
 spider.options.label.robotstxt = Parse 'robots.txt' files for new URIs

addOns/spider/src/test/java/org/zaproxy/addon/spider/SpiderParamUnitTest.java

@@ -31,6 +31,7 @@
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
 import org.junit.jupiter.params.provider.ValueSource;
 import org.mockito.MockedStatic;
 import org.parosproxy.paros.Constant;
@@ -156,4 +157,27 @@ void shouldParseDsStorePerSetting(boolean enabled) {
         // Then
         assertThat(param.isParseDsStore(), is(equalTo(enabled)));
     }
+
+    @ParameterizedTest
+    @CsvSource({"-10, 0", "-1, 0", "0, 0", "1, 1", "10, 10"})
+    void shouldLoadConfigWithMaxParseSizeBytes(int value, int expected) {
+        // Given
+        String configKey = "spider.maxParseSizeBytes";
+        configuration = new ZapXmlConfiguration();
+        configuration.setProperty(configKey, value);
+        // When
+        param.load(configuration);
+        // Then
+        assertThat(param.getMaxParseSizeBytes(), is(equalTo(expected)));
+    }
+
+    @ParameterizedTest
+    @CsvSource({"-10, 0", "-1, 0", "0, 0", "1, 1", "10, 10"})
+    void shouldSetAndPersistMaxParseSizeBytes(int value, int expected) {
+        // Given / When
+        param.setMaxParseSizeBytes(value);
+        // Then
+        assertThat(param.getMaxParseSizeBytes(), is(equalTo(expected)));
+        assertThat(configuration.getInt("spider.maxParseSizeBytes"), is(equalTo(expected)));
+    }
 }

addOns/spider/src/test/java/org/zaproxy/addon/spider/filters/DefaultParseFilterUnitTest.java

@@ -316,6 +316,19 @@ void shouldNotFilterHttpMessageWithResponseUnderMaxParseSize() throws Exception
         assertThat(filterResult.isFiltered(), is(equalTo(false)));
     }
 
+    @Test
+    void shouldNotFilterHttpMessageWithZeroMaxParseSize() throws Exception {
+        // Given
+        int maxParseSizeBytes = 0;
+        DefaultParseFilter filter =
+                new DefaultParseFilter(createSpiderParam(maxParseSizeBytes), resourceBundle);
+        HttpMessage httpMessage = createHttpMessageWithResponseBody("ABC");
+        // When
+        FilterResult filterResult = filter.filtered(httpMessage);
+        // Then
+        assertThat(filterResult.isFiltered(), is(equalTo(false)));
+    }
+
     private DefaultParseFilter createDefaultParseFilter() {
         return new DefaultParseFilter(createSpiderParam(Integer.MAX_VALUE), resourceBundle);
     }