020 subject serialization

.cursor/rules/specify-rules.mdc

@@ -19,6 +19,7 @@ Auto-generated from all feature plans. Last updated: 2025-11-04
 - N/A (text serialization/deserialization library) (014-gram-serialization)
 - Haskell (GHC 9.x) + `megaparsec` (parsing), `hspec` (testing) (016-gram-parsing-conformance)
 - N/A (In-memory data structures) (016-gram-parsing-conformance)
+- Haskell (GHC 9.8/9.10) + `megaparsec` (parsing), `containers` (Map/Set), `text` (likely for efficiency, though current code uses String) (020-subject-serialization)
 
 - (001-pattern-data-structure)
 
@@ -38,9 +39,9 @@ tests/
 : Follow standard conventions
 
 ## Recent Changes
+- 020-subject-serialization: Added Haskell (GHC 9.8/9.10) + `megaparsec` (parsing), `containers` (Map/Set), `text` (likely for efficiency, though current code uses String)
 - 019-integration-polish: Added [if applicable, e.g., PostgreSQL, CoreData, files or N/A]
 - 018-pattern-path-semantics: Added [if applicable, e.g., PostgreSQL, CoreData, files or N/A]
-- 016-gram-parsing-conformance: Added Haskell (GHC 9.x) + `megaparsec` (parsing), `hspec` (testing)
 
 
 <!-- MANUAL ADDITIONS START -->

TODO.md

@@ -35,29 +35,17 @@ Serialization and parsing for `Pattern Subject`.
 - **Parsing Conformance**: Verified 100% pass rate against `tree-sitter-gram` corpus (Feature 16).
 - **Validation**: `Gram.Validate` module implemented with duplicate definition, undefined reference, and arity checking.
 
----
-
-## 🗺️ Roadmap
+### Subject Identity & Serialization (Feature 20)
+Completed robust identity handling and round-trip capabilities.
 
-### 1. Subject Identity & Serialization (Feature 20)
-**Priority**: High / Next Up
-**Goal**: Serialize Subject instances to gram notation and parse gram notation to Subject instances, handling the identity requirement (Subject requires identity, but gram allows anonymous subjects).
-
-#### 10.1 Serialization Design
-- [ ] Design serialization format for Subject to gram notation
-- [ ] Handle anonymous subjects: gram syntax allows anonymous (unidentified) subjects, but Subject data type requires identity
-- [ ] Design strategy for assigning identity to anonymous subjects during serialization
-  - [ ] Option: Generate unique identifiers (e.g., UUIDs, sequential IDs)
-  - [ ] Option: Use placeholder identifiers that can be omitted in output
-  - [ ] Option: Track identity mapping for round-trip serialization
-- [ ] Implement `toGram :: Subject -> String` (serialize Subject to gram notation)
-- [ ] Implement `fromGram :: String -> Either ParseError Subject` (parse gram notation to Subject)
-- [ ] Write tests: verify serialization of subjects with all components
-- [ ] Write tests: verify parsing of gram notation with anonymous subjects
-- [ ] Write tests: verify round-trip serialization (parse . serialize = id, with identity handling)
+- **Identity Preservation**: Implemented sequential ID generation for anonymous subjects to ensure round-trip stability.
+- **Implicit Root**: Distinguishes between empty nodes `()` and implicit root containers `{}`.
+- **Round-Trip Verification**: Validated structural equality after serialization/deserialization cycles against the full test corpus.
 
 ---
 
+## 🗺️ Roadmap
+
 ### 2. Graph Views (Feature 21)
 **Priority**: High
 **Goal**: Interpret `Pattern` structures as different graph elements (nodes, relationships, walks) through categorical functors.

libs/gram/gram.cabal

@@ -61,6 +61,7 @@ test-suite gram-test
         text >=2.0,
         megaparsec ^>=9.6,
         hspec ^>=2.11,
+        QuickCheck ^>=2.14,
         directory ^>=1.3,
         filepath ^>=1.4
 

libs/gram/src/Gram/Parse.hs

@@ -459,6 +459,8 @@ parseGram = do
     else optional (try parseAnnotatedPattern) >>= \p -> return $ maybe [] (:[]) p
 
   additionalPatterns <- many (try (do
+    optionalSpaceWithNewlines
+    void $ optional (char ',')
     optionalSpaceWithNewlines
     nextChar <- lookAhead (satisfy (const True))
     if nextChar == '(' || nextChar == '[' || nextChar == '@'

libs/gram/src/Gram/Serialize.hs

@@ -53,8 +53,8 @@ import Data.Char (isAlpha, isAlphaNum)
 
 -- | Escape special characters in strings for gram notation.
 --
--- Escapes quotes and backslashes in string values to ensure proper
--- serialization in gram notation.
+-- Escapes quotes, backslashes, and control characters in string values 
+-- to ensure proper serialization in gram notation.
 --
 -- === Examples
 --
@@ -63,11 +63,17 @@ import Data.Char (isAlpha, isAlphaNum)
 --
 -- >>> escapeString "He said \"Hello\""
 -- "He said \\\"Hello\\\""
+--
+-- >>> escapeString "Line 1\nLine 2"
+-- "Line 1\\nLine 2"
 escapeString :: String -> String
 escapeString = concatMap escapeChar
   where
     escapeChar '"' = "\\\""
     escapeChar '\\' = "\\\\"
+    escapeChar '\n' = "\\n"
+    escapeChar '\r' = "\\r"
+    escapeChar '\t' = "\\t"
     escapeChar c = [c]
 
 -- | Format a Symbol for gram notation.
@@ -177,7 +183,7 @@ serializeLabels lbls
   | Set.null lbls = ""
   | otherwise = ":" ++ intercalate ":" (Set.toList lbls)
 
--- | Serialize a Subject to gram notation (legacy function, kept for compatibility).
+    -- | Serialize a Subject to gram notation (legacy function, kept for compatibility).
 -- Note: This always uses node syntax. Use toGram for proper syntax selection.
 serializeSubject :: Subject -> String
 serializeSubject (Subject ident lbls props) =
@@ -186,6 +192,14 @@ serializeSubject (Subject ident lbls props) =
   serializeLabels lbls ++
   serializePropertyRecord props
 
+-- | Serialize pattern elements for implicit root (no brackets/pipe).
+-- Used when serializing the top-level Gram container.
+-- NOTE: This function is now inlined into toGram to handle properties.
+-- Keeping signature for potential reuse or removing if unused.
+-- serializeImplicitElements :: [Pattern Subject] -> String
+-- serializeImplicitElements elems = 
+--   intercalate "\n" (map toGram elems)
+
 -- | Serialize pattern elements to gram notation.
 --
 -- Converts a list of Pattern Subject elements to gram notation.
@@ -257,14 +271,33 @@ serializePatternElements elems
 -- "[g | a, b]"
 toGram :: Pattern Subject -> String
 toGram p@(Pattern subj elems)
+  | isImplicitRoot subj = serializeImplicitElements (properties subj) elems -- Implicit root -> record + elements
   | null elems = serializeSubjectAsNode subj  -- No elements -> node syntax
-  | otherwise = 
-      case isWalkPattern p of
-        Just edges -> serializeWalkPattern edges
-        Nothing -> case isEdgePattern p of
-          Just (rel, left, right) -> serializeEdgePattern rel left right
-          Nothing -> serializeSubjectAsSubject subj elems  -- Has elements -> subject syntax
+  | Just edges <- isWalkPattern p = serializeWalkPattern edges
+  | Just (rel, left, right) <- isEdgePattern p = serializeEdgePattern rel left right
+  | otherwise = serializeSubjectAsSubject subj elems  -- Has elements -> subject syntax
   where
+    -- | Check if subject is the Implicit Root
+    -- Identified by "Gram.Root" label.
+    isImplicitRoot :: Subject -> Bool
+    isImplicitRoot (Subject (Symbol "") lbls _) = "Gram.Root" `Set.member` lbls
+    isImplicitRoot _ = False
+
+    -- | Serialize pattern elements for implicit root (record + elements).
+    -- Note: We do NOT serialize the "Gram.Root" label itself, as it is implicit in the file structure.
+    serializeImplicitElements :: Map String Value -> [Pattern Subject] -> String
+    serializeImplicitElements props elems = 
+      let propsStr = if Map.null props then "" else serializePropertyRecord props
+          elemsStr = intercalate "\n" (map toGram elems)
+      in case (null propsStr, null elemsStr) of
+           (True, True) -> "{}" -- Empty graph/root
+           (False, True) -> trimLeadingSpace propsStr -- Remove leading space from serializePropertyRecord
+           (True, False) -> elemsStr
+           (False, False) -> trimLeadingSpace propsStr ++ "\n" ++ elemsStr
+
+    trimLeadingSpace (' ':xs) = xs
+    trimLeadingSpace xs = xs
+
     -- | Check if pattern is a Walk Pattern: [Gram.Walk | edge1, edge2, ...]
     isWalkPattern :: Pattern Subject -> Maybe [Pattern Subject]
     isWalkPattern (Pattern (Subject _ lbls _) edges)

libs/gram/src/Gram/Transform.hs

@@ -1,3 +1,4 @@
+{-# LANGUAGE FlexibleContexts #-}
 module Gram.Transform
   ( transformGram
   ) where
@@ -10,109 +11,158 @@ import Data.Map (Map)
 import qualified Data.Map as Map
 import Data.Set (Set)
 import qualified Data.Set as Set
+import Control.Monad.State (State, evalState, get, put)
+import Data.Char (isDigit)
+
+type Transform = State Int
 
 -- | Transform a CST Gram into a Core Pattern Subject
 transformGram :: CST.Gram -> P.Pattern S.Subject
-transformGram (CST.Gram record patterns) =
+transformGram gram = evalState (transformGram' gram) (findMaxId gram + 1)
+
+-- | Find the maximum numeric suffix of IDs matching "#<N>" in the CST
+findMaxId :: CST.Gram -> Int
+findMaxId (CST.Gram _ patterns) = maximum (0 : concatMap scanPattern patterns)
+  where
+    scanPattern (CST.AnnotatedPattern _ elements) = concatMap scanElement elements
+
+    scanElement (CST.PEPath path) = scanPath path
+    scanElement (CST.PESubjectPattern sp) = scanSubjectPattern sp
+    scanElement (CST.PEReference ident) = scanIdentifier (Just ident)
+
+    scanPath (CST.Path startNode segments) = 
+      scanNode startNode ++ concatMap scanSegment segments
+
+    scanSegment (CST.PathSegment rel nextNode) = 
+      scanRelationship rel ++ scanNode nextNode
+
+    scanNode (CST.Node subjData) = scanSubjectData subjData
+
+    scanRelationship (CST.Relationship _ subjData) = scanSubjectData subjData
+
+    scanSubjectPattern (CST.SubjectPattern subjData nested) = 
+      scanSubjectData subjData ++ concatMap scanElement nested
+
+    scanSubjectData Nothing = []
+    scanSubjectData (Just (CST.SubjectData ident _ _)) = scanIdentifier ident
+
+    scanIdentifier (Just (CST.IdentSymbol (CST.Symbol s))) = case parseGeneratedId s of
+      Just n -> [n]
+      Nothing -> []
+    scanIdentifier _ = []
+
+    parseGeneratedId ('#':rest) | all isDigit rest && not (null rest) = Just (read rest)
+    parseGeneratedId _ = Nothing
+
+transformGram' :: CST.Gram -> Transform (P.Pattern S.Subject)
+transformGram' (CST.Gram record patterns) =
   case (record, patterns) of
     (Just props, []) -> 
       -- Only record
-      P.Pattern (S.Subject (S.Symbol "") Set.empty props) []
-    (Just props, pats) ->
-      -- Record + Patterns: Record becomes root properties, patterns become elements
-      P.Pattern (S.Subject (S.Symbol "") Set.empty props) (map transformPattern pats)
+      return $ P.Pattern (S.Subject (S.Symbol "") (Set.singleton "Gram.Root") props) []
+    (Just props, pats) -> do
+      pats' <- mapM transformPattern pats
+      -- Explicit root record.
+      return $ P.Pattern (S.Subject (S.Symbol "") (Set.singleton "Gram.Root") props) pats'
     (Nothing, [p]) ->
-      -- Single pattern (common case)
       transformPattern p
-    (Nothing, pats) ->
-      -- Multiple patterns without root record: wrap in implicit root
-      P.Pattern (S.Subject (S.Symbol "") Set.empty Map.empty) (map transformPattern pats)
+    (Nothing, pats) -> do
+      pats' <- mapM transformPattern pats
+      return $ P.Pattern (S.Subject (S.Symbol "") (Set.singleton "Gram.Root") Map.empty) pats'
 
-transformPattern :: CST.AnnotatedPattern -> P.Pattern S.Subject
+transformPattern :: CST.AnnotatedPattern -> Transform (P.Pattern S.Subject)
 transformPattern (CST.AnnotatedPattern _ elements) =
   case elements of
     [el] -> transformElement el
-    (first:rest) -> 
-      -- Multiple elements: First acts as container/root of the sequence (legacy behavior preserved)
-      -- OR we should wrap them? The existing parser treated comma-separated lists by nesting:
-      -- "a, b" -> Pattern a [Pattern b]
-      -- Let's preserve this for now to pass tests.
-      let root = transformElement first
-          others = map transformElement rest
-      in P.Pattern (P.value root) (P.elements root ++ others)
-    [] -> P.Pattern (S.Subject (S.Symbol "") Set.empty Map.empty) []
-
-transformElement :: CST.PatternElement -> P.Pattern S.Subject
+    (first:rest) -> do
+      root <- transformElement first
+      others <- mapM transformElement rest
+      return $ P.Pattern (P.value root) (P.elements root ++ others)
+    [] -> return $ P.Pattern (S.Subject (S.Symbol "") Set.empty Map.empty) []
+
+transformElement :: CST.PatternElement -> Transform (P.Pattern S.Subject)
 transformElement (CST.PEPath path) = transformPath path
 transformElement (CST.PESubjectPattern b) = transformSubjectPattern b
-transformElement (CST.PEReference ident) =
-  P.Pattern (S.Subject (transformIdentifier (Just ident)) Set.empty Map.empty) []
+transformElement (CST.PEReference ident) = do
+  sym <- transformIdentifier (Just ident)
+  return $ P.Pattern (S.Subject sym Set.empty Map.empty) []
 
 -- | Transform a path into a Pattern.
 -- 
 -- 1. Single Node: (a) -> Pattern a []
 -- 2. Single Edge: (a)-[r]->(b) -> Pattern r [a, b]
 -- 3. Walk: (a)-[r1]->(b)-[r2]->(c) -> Pattern walk [Pattern r1 [a, b], Pattern r2 [b, c]]
-transformPath :: CST.Path -> P.Pattern S.Subject
+transformPath :: CST.Path -> Transform (P.Pattern S.Subject)
 transformPath (CST.Path startNode segments) =
   case segments of
     [] -> transformNode startNode
-    [seg] -> 
+    [seg] -> do
       -- Single Edge case: Return the edge pattern directly
       -- (a)-[r]->(b) becomes [r | a, b]
-      let left = transformNode startNode
-          right = transformNode (CST.segmentNode seg)
-          rel = transformRelationship (CST.segmentRel seg)
-      in P.Pattern (P.value rel) [left, right]
-    _ -> 
+      left <- transformNode startNode
+      right <- transformNode (CST.segmentNode seg)
+      rel <- transformRelationship (CST.segmentRel seg)
+      return $ P.Pattern (P.value rel) [left, right]
+    _ -> do
       -- Walk case (multiple segments): Return a Walk Pattern containing edges
       -- (a)-[r1]->(b)-[r2]->(c) becomes [walk | [r1 | a, b], [r2 | b, c]]
-      let edges = constructWalkEdges startNode segments
-          -- Use a specific label for Walk container to distinguish it
-          walkSubject = S.Subject (S.Symbol "") (Set.singleton "Gram.Walk") Map.empty
-      in P.Pattern walkSubject edges
+      leftP <- transformNode startNode
+      edges <- constructWalkEdges leftP segments
+      -- Use a specific label for Walk container to distinguish it
+      let walkSubject = S.Subject (S.Symbol "") (Set.singleton "Gram.Walk") Map.empty
+      return $ P.Pattern walkSubject edges
 
 -- | Construct a list of Edge Patterns from a start node and path segments.
-constructWalkEdges :: CST.Node -> [CST.PathSegment] -> [P.Pattern S.Subject]
-constructWalkEdges _ [] = []
-constructWalkEdges leftNode (seg:rest) =
+-- We pass the transformed left pattern to ensure identity continuity in the walk.
+constructWalkEdges :: P.Pattern S.Subject -> [CST.PathSegment] -> Transform [P.Pattern S.Subject]
+constructWalkEdges _ [] = return []
+constructWalkEdges leftP (seg:rest) = do
   let rightNode = CST.segmentNode seg
-      leftP = transformNode leftNode
-      rightP = transformNode rightNode
-      relP = transformRelationship (CST.segmentRel seg)
-      -- Create self-contained edge: [rel | left, right]
-      edge = P.Pattern (P.value relP) [leftP, rightP]
-  in edge : constructWalkEdges rightNode rest
-
-transformNode :: CST.Node -> P.Pattern S.Subject
-transformNode (CST.Node subjData) =
-  let subj = maybe emptySubject transformSubjectData subjData
-  in P.Pattern subj []
-
-transformSubjectPattern :: CST.SubjectPattern -> P.Pattern S.Subject
-transformSubjectPattern (CST.SubjectPattern subjData nested) =
-  let subj = maybe emptySubject transformSubjectData subjData
-      nestedPats = map transformElement nested
-  in P.Pattern subj nestedPats
-
-transformRelationship :: CST.Relationship -> P.Pattern S.Subject
-transformRelationship (CST.Relationship _ subjData) =
+  rightP <- transformNode rightNode
+  relP <- transformRelationship (CST.segmentRel seg)
+  -- Create self-contained edge: [rel | left, right]
+  let edge = P.Pattern (P.value relP) [leftP, rightP]
+  restEdges <- constructWalkEdges rightP rest
+  return (edge : restEdges)
+
+transformNode :: CST.Node -> Transform (P.Pattern S.Subject)
+transformNode (CST.Node subjData) = do
+  subj <- maybe transformEmptySubject transformSubjectData subjData
+  return $ P.Pattern subj []
+
+transformSubjectPattern :: CST.SubjectPattern -> Transform (P.Pattern S.Subject)
+transformSubjectPattern (CST.SubjectPattern subjData nested) = do
+  subj <- maybe transformEmptySubject transformSubjectData subjData
+  nestedPats <- mapM transformElement nested
+  return $ P.Pattern subj nestedPats
+
+transformRelationship :: CST.Relationship -> Transform (P.Pattern S.Subject)
+transformRelationship (CST.Relationship _ subjData) = do
   -- Arrow string is currently ignored in Pattern Subject (as per design)
-  let subj = maybe emptySubject transformSubjectData subjData
-  in P.Pattern subj []
+  subj <- maybe transformEmptySubject transformSubjectData subjData
+  return $ P.Pattern subj []
 
-transformSubjectData :: CST.SubjectData -> S.Subject
-transformSubjectData (CST.SubjectData ident labels props) =
-  S.Subject
-    (transformIdentifier ident)
+transformSubjectData :: CST.SubjectData -> Transform S.Subject
+transformSubjectData (CST.SubjectData ident labels props) = do
+  sym <- transformIdentifier ident
+  return $ S.Subject
+    sym
     labels
     props
 
-transformIdentifier :: Maybe CST.Identifier -> S.Symbol
-transformIdentifier Nothing = S.Symbol ""
-transformIdentifier (Just (CST.IdentSymbol (CST.Symbol s))) = S.Symbol s
-transformIdentifier (Just (CST.IdentString s)) = S.Symbol s
-transformIdentifier (Just (CST.IdentInteger i)) = S.Symbol (show i)
+transformIdentifier :: Maybe CST.Identifier -> Transform S.Symbol
+transformIdentifier Nothing = generateId
+transformIdentifier (Just (CST.IdentSymbol (CST.Symbol s))) = return $ S.Symbol s
+transformIdentifier (Just (CST.IdentString s)) = return $ S.Symbol s
+transformIdentifier (Just (CST.IdentInteger i)) = return $ S.Symbol (show i)
+
+transformEmptySubject :: Transform S.Subject
+transformEmptySubject = do
+  sym <- generateId
+  return $ S.Subject sym Set.empty Map.empty
 
-emptySubject :: S.Subject
-emptySubject = S.Subject (S.Symbol "") Set.empty Map.empty
+generateId :: Transform S.Symbol
+generateId = do
+  i <- get
+  put (i + 1)
+  return $ S.Symbol ("#" ++ show i)