Some Improvements + Fix Data Race

.github/workflows/build.yml

@@ -15,7 +15,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Run Go tests
-        run: cd backend && go test -cover -v ./...
+        run: cd backend && go test -cover -race -v ./...
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3

README.md

@@ -35,20 +35,22 @@ Sloggo is made by [Phare](https://phare.io), a small bootstrapped company buildi
 
 1. Start the container with docker or podman:
 
-```bash
-docker run -p 5514:5514/udp -p 6514:6514 -p 8080:8080 \
-  -e SLOGGO_LISTENERS=tcp,udp \
-  -e SLOGGO_UDP_PORT=5514 \
-  -e SLOGGO_TCP_PORT=6514 \
-  -e SLOGGO_API_PORT=8080 \
-   ghcr.io/phare/sloggo:latest
-```
+   ```bash
+   docker run --name sloggo \
+      -p 5514:5514/udp -p 6514:6514 -p 8080:8080 \
+      -e SLOGGO_LISTENERS=tcp,udp \
+      -e SLOGGO_UDP_PORT=5514 \
+      -e SLOGGO_TCP_PORT=6514 \
+      -e SLOGGO_API_PORT=8080 \
+      -v ./data:/app/.duckdb \
+      ghcr.io/phare/sloggo:latest
+   ```
 
 2. Send some logs
 
-```bash
-echo "<34>1 2025-08-04T12:00:00Z myhost sloggo - - - Hello, Sloggo" | nc localhost 6514
-```
+   ```bash
+   echo "<34>1 2025-08-04T12:00:00Z myhost sloggo - - - Hello, Sloggo" | nc localhost 6514
+   ```
 
 3. Access the application:
    - Frontend: [http://localhost:8080/](http://localhost:8080/)

backend/Makefile

@@ -0,0 +1,69 @@
+GO=go
+GOTEST=$(GO) test
+BIN_DIR=bin
+EXPORT_RESULT?=FALSE
+.DEFAULT_GOAL:=help
+
+##################################### Binary #####################################
+
+.PHONY: pre
+pre: ## Create the bin directory
+	@mkdir -p $(BIN_DIR)
+
+.PHONY: clean
+clean: ## Clean the bin directory
+	@rm -rf $(BIN_DIR)
+	@rm -f ./coverage.out
+
+.PHONY: build
+build: pre ## Create the main binary
+	@CGO_ENABLED=1 $(GO) build -ldflags="-s -w" -o $(BIN_DIR)/sloggo main.go
+
+##################################### Beautify #####################################
+
+.PHONY: format
+format: ## Format the source code
+	@find . -name '*.go' -not -path './vendor/*' | xargs -n1 go fmt
+
+.PHONY: lint
+lint:
+	@$(GO) install github.com/golangci/golangci-lint/v2/cmd/[email protected]
+	@golangci-lint run --timeout 3m0s
+
+##################################### Test #####################################
+
+.PHONY: goconvey
+goconvey: ## Run goconvey
+	@$(GO) install github.com/smartystreets/goconvey@latest
+	@goconvey -port 1234
+
+.PHONY: test
+test: ## Run the tests
+ifeq ($(EXPORT_RESULT), TRUE)
+	@$(GO) install github.com/jstemmer/go-junit-report/v2@latest
+	@$(eval OUTPUT_OPTIONS = | go-junit-report -iocopy -set-exit-code -out junit-report.xml)
+endif
+	@$(GO) clean -testcache
+	@$(GOTEST) -v -race ./... $(OUTPUT_OPTIONS)
+
+.PHONY: coverage
+coverage: $(wildcard **/**/*.go) ## Run the tests and generate the coverage reports
+	@$(GOTEST) -cover -covermode=count -coverprofile=coverage.out ./...
+ifeq ($(EXPORT_RESULT), TRUE)
+	@$(GO) install github.com/axw/gocov/gocov@latest
+	@$(GO) install github.com/AlekSi/gocov-xml@latest
+	@$(GO) tool cover -html=coverage.out -o coverage.html
+	@gocov convert coverage.out | gocov-xml > coverage.xml
+	@rm -f coverage.out
+endif
+
+.PHONY: vulnerability-scan ## Scan the project for known vulnerabilities
+vulnerability-scan:
+	@$(GO) install github.com/google/osv-scanner/v2/cmd/osv-scanner@latest
+	@osv-scanner scan ./
+
+##################################### Help #####################################
+
+.PHONY: help
+help: ## Show this help
+	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' Makefile | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}'

backend/db/store.go

@@ -125,16 +125,22 @@ func StoreLog(entry models.LogEntry) error {
 
 	// If we've reached the max batch size, process immediately
 	if len(batchLogs) >= maxBatchStoreLogsSize {
+		// Don't unlock here - let ProcessBatchStoreLogs handle it
+		// by calling it while holding the lock
+		entries := batchLogs
+		batchLogs = make([]models.LogEntry, 0, maxBatchStoreLogsSize)
 		batchLogsMutex.Unlock()
-		return ProcessBatchStoreLogs()
+
+		// Process the batch outside the lock
+		return processBatchStoreLogsWithEntries(entries)
 	}
 
 	batchLogsMutex.Unlock()
 	return nil
 }
 
-// processBatchStoreLogsUnsafe processes all pending log entries without acquiring mutex
-// Must be called with batchStoreLogsMutex already held
+// ProcessBatchStoreLogs processes all pending log entries
+// This is called by the periodic batch processor
 func ProcessBatchStoreLogs() error {
 	batchLogsMutex.Lock()
 	if len(batchLogs) == 0 {
@@ -143,10 +149,19 @@ func ProcessBatchStoreLogs() error {
 	}
 
 	entries := batchLogs
-	batchLogs = batchLogs[:0]
-
+	batchLogs = make([]models.LogEntry, 0, maxBatchStoreLogsSize)
 	batchLogsMutex.Unlock()
 
+	return processBatchStoreLogsWithEntries(entries)
+}
+
+// processBatchStoreLogsWithEntries processes a batch of log entries
+// This function does not touch the global batchLogs slice
+func processBatchStoreLogsWithEntries(entries []models.LogEntry) error {
+	if len(entries) == 0 {
+		return nil
+	}
+
 	// Get the underlying DuckDB connection from sql.DB
 	dbConn, err := db.Conn(context.Background())
 	if err != nil {