removing unused setting and updating Readme

Author: Christian Förster

.gitignore

@@ -44,4 +44,6 @@ go.work.sum
 
 .DS_Store
 
-*.html
+.claude
+
+*.html

ReadMe.md

@@ -10,8 +10,10 @@ Currently only a few checks are implemented:
 - HasNoWhiteSpace (for filenames)
 - IsFreeOfKeywords (checking file contents); non binary, .xlsx and .docx are supported
 - IsValidName (checking if nonsense files are present eg: .Rhistory)
+- HasFileNameSpecialChars (~!?@#$%^&*`;,'"()<>[]{})
+- IsFileNameTooLong (>64 is too long)
 
-Archives (.zip, .tar, .7z) are also supported. On these the content (IsFreeOfKeywords) on each file is **not** checked.
+Archives (.zip, .tar, .7z) are also supported. On these the content (IsFreeOfKeywords) on each file is checked if the file is not too big.
 As *.tar.gz* files require complete unpacking of the archive to access the list of contained files it is not supported as it would be too slow for large archives.
 
 **By respository:**
@@ -87,6 +89,21 @@ or you compile first and run via:
 pc -location your-ckan-package-name
 ```
 
+run with Terminal User Interface:
+```bash
+pc -config pc.toml -location .  --tui
+```
+
+run with html output:
+```bash
+pc -config pc.toml -location .  --html report.html
+```
+
+run with plain output:
+```bash
+pc -config pc.toml -location .  --plain
+```
+
 ## Building
 To build (https://github.com/confluentinc/confluent-kafka-go/issues/1092#issuecomment-2373681430): 
 ```bash
@@ -159,8 +176,3 @@ ssh -t user@remote-server "TERM=xterm-256color LANG=en_US.UTF-8 cd /path/to/pc &
 go test ./...
 ```
 
-## Next steps
-
-I added to new checks in the checks by file file. Please include these in the config and add white and blacklist functionalities as requires. I believe it should already be implemented. Please hten add the corresponding sdections to the 3 toml files. 
-
-After adding the tests please check the functionality of pc. I belive there must be a bug. Running this go run . -config pc.toml -location testdata/archives should find a lot of problem but it is not. the data in the different archive formats is identical. So it should always find the same for zip, tar and 7z. What is going on?

pc.toml.example

@@ -39,8 +39,6 @@
 #####################################################################################
 
 [general]
-# Maximum number of messages of the same type before truncating
-maxMessagesPerType = 5
 # Maximum size for individual files in archives (bytes) - 10MB
 maxArchiveFileSize = 10485760
 # Maximum total memory for archive processing (bytes) - 500MB
@@ -70,13 +68,10 @@ whitelist = []
 keywordArguments = [
     { keywords = ["password", "secret", "key", "token", "api", "credential", "auth"], info = "Security credentials detected" },
     { keywords = ["id_rsa", "id_ed25519", "BEGIN PRIVATE KEY", "BEGIN RSA PRIVATE KEY"], info = "Private key detected" },
-    { keywords = ["AKIAI", "AKIA", "sk_live", "sk_test", "xoxb", "xoxp"], info = "API key detected" },
     { keywords = ["jwt", "bearer", "oauth", "client_secret"], info = "Authentication token detected" },
     { keywords = ["database", "db_password", "connection_string"], info = "Database credentials detected" },
-    { keywords = ["stripe", "paypal", "sendgrid", "twilio", "slack"], info = "Third-party service credentials" },
-    { keywords = ["/Users/", "C:\\Users\\", "Q:"], info = "Hardcoded file paths detected" },
-    { keywords = ["admin", "root", "superuser"], info = "Administrative accounts detected" },
-    { keywords = ["ssn", "social_security", "credit_card", "phone"], info = "PII detected" }
+    { keywords = ["/home/", "/Users/", "C:\\Users\\", "Q:"], info = "Hardcoded file paths detected" },
+    { keywords = ["admin", "root", "superuser"], info = "Administrative accounts detected" }
 ]
 
 [test.HasFileNameSpecialChars]
@@ -104,7 +99,7 @@ keywordArguments = [
         ".Rbuildignore", "__pycache__", ".vscode", 
         ".ipynb_checkpoints", "venv", ".idea", ".egg-info", 
         ".pytest_cache", ".pyc", ".tox", ".python_version", 
-        ".coverage", ".benchmark", ".doc", ".xls"
+        ".coverage", ".benchmark", ".doc", ".xls", ".DS_Store"
     ]}
 ]
 

pkg/config/config_parser.go

@@ -23,7 +23,6 @@ type OperationConfig struct {
 }
 
 type GeneralConfig struct {
-	MaxMessagesPerType     int
 	MaxArchiveFileSize     int64 // Maximum size for individual files in archives (bytes)
 	MaxTotalArchiveMemory  int64 // Maximum total memory for archive processing (bytes)
 	MaxContentScanFileSize int64 // Maximum size for files that read content (like IsFreeOfKeywords) (bytes)
@@ -45,7 +44,6 @@ func ParseConfig(filename string) (*Config, error) {
 
 	c := &Config{
 		General: &GeneralConfig{
-			MaxMessagesPerType:     10,                      // Default value
 			MaxArchiveFileSize:     10 * 1024 * 1024,       // 10MB default
 			MaxTotalArchiveMemory:  100 * 1024 * 1024,      // 100MB default
 			MaxContentScanFileSize: 1024 * 1024 * 1024,     // 1GB default for content scanning
@@ -86,9 +84,6 @@ func ParseConfig(filename string) (*Config, error) {
 
 	// Parse general section
 	if generalData, ok := raw["general"].(map[string]interface{}); ok {
-		if maxMsgs, ok := generalData["maxMessagesPerType"].(int64); ok {
-			c.General.MaxMessagesPerType = int(maxMsgs)
-		}
 		if maxArchiveFileSize, ok := generalData["maxArchiveFileSize"].(int64); ok {
 			c.General.MaxArchiveFileSize = maxArchiveFileSize
 		}

pkg/utils/archive_parallel_test.go

@@ -7,17 +7,16 @@ import (
 	"github.com/eawag-rdm/pc/pkg/structs"
 )
 
-// TestArchiveParallelProcessing tests that all archives of the same type 
+// TestArchiveParallelProcessing tests that all archives of the same type
 // (with different extensions) are processed and their messages are preserved.
 // This test ensures the message truncation bug doesn't regress.
 func TestArchiveParallelProcessing(t *testing.T) {
 	// Create test configuration similar to production
 	cfg := config.Config{
 		General: &config.GeneralConfig{
-			MaxMessagesPerType:        5,
-			MaxArchiveFileSize:        10485760, // 10MB
-			MaxTotalArchiveMemory:     536870912, // 500MB
-			MaxContentScanFileSize:    20971520, // 20MB
+			MaxArchiveFileSize:     10485760,  // 10MB
+			MaxTotalArchiveMemory:  536870912, // 500MB
+			MaxContentScanFileSize: 20971520,  // 20MB
 		},
 		Tests: map[string]*config.TestConfig{
 			"IsFreeOfKeywords": {
@@ -81,57 +80,3 @@ func TestArchiveParallelProcessing(t *testing.T) {
 		t.Errorf("Expected %d unique archives, found %d", expectedArchiveCount, len(archiveNames))
 	}
 }
-
-// TestArchiveProcessingWithoutTruncation verifies that disabling message truncation
-// preserves all archive messages, which was the root cause of the original bug.
-func TestArchiveProcessingWithoutTruncation(t *testing.T) {
-	cfg := config.Config{
-		General: &config.GeneralConfig{
-			MaxMessagesPerType: 2, // Very low limit to trigger truncation if enabled
-		},
-		Tests: map[string]*config.TestConfig{
-			"IsFreeOfKeywords": {
-				KeywordArguments: []map[string]interface{}{
-					{"keywords": []string{"test"}, "info": "Test keyword"},
-				},
-			},
-		},
-	}
-
-	// Create multiple archive files that would trigger truncation
-	files := []structs.File{
-		{Path: "archive1.zip", Name: "archive1.zip", IsArchive: true},
-		{Path: "archive2.zip", Name: "archive2.zip", IsArchive: true}, 
-		{Path: "archive3.zip", Name: "archive3.zip", IsArchive: true},
-		{Path: "archive4.zip", Name: "archive4.zip", IsArchive: true},
-		{Path: "archive5.zip", Name: "archive5.zip", IsArchive: true},
-	}
-
-	// Mock check that returns archive-like messages (which would be grouped together)
-	mockCheck := func(file structs.File, config config.Config) []structs.Message {
-		return []structs.Message{
-			{
-				Content: "Test keyword 'test'. In archived file: 'test.txt'",
-				Source:  file,
-			},
-		}
-	}
-
-	// Use ApplyChecksFilteredByFileOnArchive directly to test our specific function
-	messages := ApplyChecksFilteredByFileOnArchive(cfg, []func(structs.File, config.Config) []structs.Message{mockCheck}, files)
-
-	// All 5 messages should be preserved (no truncation)
-	if len(messages) != 5 {
-		t.Errorf("Expected 5 messages without truncation, got %d", len(messages))
-	}
-
-	// Verify all archives are represented
-	uniqueArchives := make(map[string]bool)
-	for _, msg := range messages {
-		uniqueArchives[msg.Source.(structs.File).Name] = true
-	}
-
-	if len(uniqueArchives) != 5 {
-		t.Errorf("Expected messages from 5 different archives, got %d", len(uniqueArchives))
-	}
-}