add an IL with a dedicated pointer type (#172)

## Summary

Add the `LPtr` IL providing a dedicated pointer type and an `Offset`
operation for pointer arithmetic. Higher-level ILs as well as `skully`
and `source2il` are changed to use the new `Ptr` type.

## Details

### Idea

The idea behind a dedicated pointer type is to:
1. allow for better static error detection, once type checking for ILs
   is implemented
2. allow higher-level passes to defer decision making (i.e., regarding
   how a pointer is implemented)
3. keep the type for code generators that need it

### Implementation

* add the `LPtr` providing the `Ptr` type and `Nil` and `Offset`
  expression. It extends the `L0` IL and is the new basis for the
  `L1` IL 
* integrate the new IL and the new `ptrToInt` pass (`LPtr` -> `L0`)
  into the main executable and the REPL 
* update the higher-level ILs to consider `Ptr` and `Nil`
* change the stack allocation pass to emit `Offset` instead of integer
  arithmetic
* change the aggregate-related passes, as well as the global-to-pointer
  pass, to use `Ptr` instead of lowering to `UInt`
* update the expected output of the modified existing tests
* add tests for the new `ptrToInt` pass 

The IL uses a symbolic name (rather than a numeric one) so that the
existing ILs don't have to be renamed.

Usage of `Offset` is disallowed past the `L2` IL for now, so that raw
pointer arithmetic is not exposed to higher-level ILs for now.

### Source2IL

* add an internal-only `tkPointer` type
* replace the `pointerType` constant with using `tkPointer` directly
* translate `tkPointer` and procedure pointers to the IL `Ptr` type
* update the hand-written `L30` code to not use pointer arithmetic

### Skully

* translate MIR pointer-like types to `Ptr`
* use `Nil` for representing nil pointers
* adjust the cast translation, which needs to handle pointers no longer
  being arithmetic types

Author: zerbina

languages/lang1.md

@@ -1,14 +1,15 @@
 ## L1 Language
 
 ```grammar
-.extends lang0
+.extends langPtr
 ```
 
-Numeric types can be identified types and identified types can be used in all
-places where types are expected.
+Numeric and pointer types can be identified types and identified types can be
+used in all places where types are expected.
 
 ```grammar
 typedesc += <numtype>
+          | (Ptr)
 type += <type_id>
 ```
 

languages/lang3.md

@@ -56,3 +56,9 @@ expr += (Addr <path>)
       | (Copy <path>)
 stmt += (Asgn <path> <expr>)
 ```
+
+Pointer arithmetic is not allowed anymore.
+
+```grammar
+expr -= (Offset <expr> <expr> <intVal>)
+```

languages/langPtr.md

@@ -0,0 +1,26 @@
+## LPtr
+
+```grammar
+.extends lang0
+```
+
+Pointer types are introduced.
+
+```grammar
+type += (Ptr)
+```
+
+The pointer value representing a pointer that doesn't point anywhere is
+represented by the `Nil` expression.
+
+```grammar
+expr += (Nil)
+```
+
+Pointer arithmetic is done using the `Offset` operation, which applies an
+unsigned integer with the same byte-width as a pointer scaled by a positive,
+non-zero constant value.
+
+```grammar
+expr += (Offset base:<expr> idx:<expr> scale:<intVal>)
+```

passes/pass_aggregateParams.nim

@@ -25,7 +25,6 @@ type
   TypeId = uint32
 
   Context = object
-    addrType: Node
     cache: seq[NodeIndex]
       ## indexed by procedure IDs. Caches the signature type for every
       ## procedure. This greatly speeds up type computation of calls, as it
@@ -49,6 +48,8 @@ type
     params: PackedSet[LocalId]
       ## the parameters that are turned into pointers
 
+const PtrType = Node(kind: Ptr)
+
 using
   c: var Context
   tree: PackedTree[NodeKind]
@@ -101,7 +102,7 @@ proc typeof(c; tree; n): Node =
     unreachable()
 
 proc newLocal(c; typ: Node): Node =
-  assert typ.kind in {Type, UInt, Int, Float}
+  assert typ.kind in {Type, UInt, Int, Float, Ptr}
   result = Node(kind: Local, val: uint32(c.firstTemp + c.temps.len))
   c.temps.add typ
 
@@ -168,7 +169,7 @@ proc lowerOperand(c; tree; n; reference: int, bu) =
     # code and/or more work for later passes (more locals usually equals more
     # work and bookkeeping).
     case tree[n].kind
-    of IntVal, FloatVal, ProcVal:
+    of IntVal, FloatVal, ProcVal, Nil:
       discard "nothing to do"
     of Addr:
       c.lowerAddr(tree, n, reference, bu)
@@ -220,7 +221,7 @@ proc lowerCallArgs(c; tree; n; start: int, last: BackwardsIndex; bu) =
 
 proc lowerExpr(c; tree; n, bu) =
   case tree[n].kind
-  of IntVal, FloatVal, ProcVal:
+  of IntVal, FloatVal, ProcVal, Nil:
     discard "nothing to do"
   of Copy:
     let x = tree.child(n, 0)
@@ -362,7 +363,7 @@ proc lowerProc(c; tree; n; sig: NodeIndex, bu) =
 
   if isAggregate(tree, tree[sig, 0]):
     c.hasOutParam = true
-    c.outParam = c.newLocal(c.addrType)
+    c.outParam = c.newLocal(PtrType)
   else:
     c.hasOutParam = false
 
@@ -400,29 +401,29 @@ proc lowerProc(c; tree; n; sig: NodeIndex, bu) =
     bu.modifyTree tree, c.locals, list:
       # update the parameter locals:
       for it in c.params.items:
-        bu.replace tree.child(c.locals, it.ord), c.addrType
+        bu.replace tree.child(c.locals, it.ord), PtrType
 
       # append the new locals to the list of locals:
       let last = tree.fin(c.locals)
       for it in c.temps.items:
         bu.insert list, last, it
 
-proc lower*(tree; ptrSize: int): ChangeSet[NodeKind] =
+proc lower*(tree): ChangeSet[NodeKind] =
   ## Computes the changeset representing the lowering for a whole module
-  ## (`tree`). `ptrSize` is the size-in-bytes of a pointer value.
-  var c = Context(addrType: Node(kind: UInt, val: ptrSize.uint32))
+  ## (`tree`).
+  var c = Context()
 
   for it in tree.items(tree.child(0)):
     if tree[it].kind == ProcTy:
       result.modifyTree tree, it, m:
         # turn an aggregate return value into an out parameter:
         if isAggregate(tree, tree[it, 0]):
           result.replace tree.child(it, 0), result.buildTree(tree(Void))
-          result.insert m, tree.fin(it), c.addrType
+          result.insert m, tree.fin(it), PtrType
 
         for x in tree.items(it, 1):
           if isAggregate(tree, tree[x]):
-            result.replace x, c.addrType
+            result.replace x, PtrType
 
   let procs = tree.child(2)
 

passes/pass_aggregatesToBlob.nim

@@ -5,9 +5,8 @@ import
   passes/[changesets, syntax, trees]
 
 type
-  Node = TreeNode[NodeKind]
   Context = object
-    addrType: Node
+    ptrSize: int
     # per-procedure state:
     locals: NodeIndex
 
@@ -29,11 +28,12 @@ proc resolve(tree; n): NodeIndex =
   else:
     n
 
-proc size(tree; n: NodeIndex): uint =
+proc size(c; tree; n: NodeIndex): uint =
   ## `n` must be the node index of a type description.
   case tree[n].kind
   of Record, Union, Array: tree[n, 0].val.uint
   of Int, UInt, Float:     tree[n].val.uint
+  of Ptr:                  c.ptrSize.uint
   else:                    unreachable()
 
 proc alignment(tree; n: NodeIndex): uint =
@@ -42,12 +42,12 @@ proc alignment(tree; n: NodeIndex): uint =
   of Record, Union, Array: tree[n, 1].val.uint
   else:                    unreachable()
 
-proc elementOffset(tree; n: NodeIndex, elem: uint32): uint =
+proc elementOffset(c; tree; n: NodeIndex, elem: uint32): uint =
   ## `n` must be the node index of a type description.
   case tree[n].kind
   of Union:  0'u
   of Record: tree[tree.child(n, elem + 2), 0].val
-  of Array:  size(tree, tree.resolve(tree.child(n, 3))) * elem
+  of Array:  size(c, tree, tree.resolve(tree.child(n, 3))) * elem
   else:      unreachable()
 
 proc typeOfElem(tree; n: NodeIndex, elem: uint32): NodeIndex =
@@ -76,11 +76,11 @@ proc lowerPath(c; tree; n; bu): VirtualTree =
   for it in tree.items(n, 2):
     if tree[it].kind == Immediate:
       # a static field or array access
-      offset += elementOffset(tree, typ, tree[it].val)
+      offset += elementOffset(c, tree, typ, tree[it].val)
       typ = tree.resolve(typeOfElem(tree, typ, tree[it].val))
     elif tree[it].kind == IntVal:
       # a static array access
-      offset += elementOffset(tree, typ, tree.getInt(it).uint32)
+      offset += elementOffset(c, tree, typ, tree.getInt(it).uint32)
       typ = tree.resolve(typeOfElem(tree, typ, 0))
     else:
       # an array access with a dynamic index
@@ -89,26 +89,27 @@ proc lowerPath(c; tree; n; bu): VirtualTree =
         #       adding new packed numbers
         # add the static offset computed so far:
         result = bu.buildTree:
-          tree(Add, node(c.addrType),
+          tree(Offset,
             embed(result),
-            node(IntVal, offset.uint32))
+            node(IntVal, offset.uint32),
+            node(IntVal, 1))
         offset = 0
 
       typ = tree.resolve(typeOfElem(tree, typ, 0))
 
       # apply the dynamic array element offset:
       result = bu.buildTree:
-        tree(Add, node(c.addrType),
+        tree(Offset,
           embed(result),
-          tree(Mul, node(c.addrType),
-            embed(c.loweredExpr(tree, it, bu)),
-            node(IntVal, size(tree, typ).uint32)))
+          embed(c.loweredExpr(tree, it, bu)),
+          node(IntVal, size(c, tree, typ).uint32))
 
   if offset > 0:
     result = bu.buildTree:
-      tree(Add, node(c.addrType),
+      tree(Offset,
         embed(result),
-        node(IntVal, offset.uint32))
+        node(IntVal, offset.uint32),
+        node(IntVal, 1))
 
 proc lowerExpr(c; tree; n; bu) =
   case tree[n].kind
@@ -153,26 +154,26 @@ proc lowerStmt(c; tree; n; bu) =
     # no statement-specific transformation, just lower the expressions within
     c.lowerExpr(tree, n, bu)
 
-proc lower*(tree; ptrSize: int): ChangeSet[NodeKind] =
+proc lower*(tree; ptrSize: Positive): ChangeSet[NodeKind] =
   ## Computes the changeset representing the lowering for a whole module
   ## (`tree`). `ptrSize` is the size-in-bytes of a pointer value.
 
   # turn all aggregate types into blob types:
   for it in tree.items(tree.child(0)):
     if tree[it].kind in {Record, Union, Array}:
+      let c = Context(ptrSize: ptrSize)
       result.replace it:
         result.buildTree:
           tree(Blob,
-            node(Immediate, uint32 size(tree, it)),
+            node(Immediate, uint32 size(c, tree, it)),
             node(Immediate, uint32 alignment(tree, it)))
 
   # lower the procedures:
   for it in tree.items(tree.child(2)):
     if tree[it].kind == ProcDef:
       let
         (_, locals, body) = tree.triplet(it)
-        c = Context(locals: locals,
-                    addrType: Node(kind: UInt, val: uint32 ptrSize))
+        c = Context(ptrSize: ptrSize, locals: locals)
 
       for blk in tree.items(body):
         for s in tree.items(blk, 1):

passes/pass_globalsToPointer.nim

@@ -7,9 +7,10 @@ import
   passes/[changesets, syntax, trees]
 
 type
+  Node = TreeNode[NodeKind]
   Context = object
-    ptrSize: int
-    globals: Table[uint32, TreeNode[NodeKind]]
+    ptrSize: uint32
+    globals: Table[uint32, Node]
       ## global ID -> global's type. Only keeps a mapping for globals that
       ## refer to addressable locations
 
@@ -19,14 +20,16 @@ using
   n: NodeIndex
   bu: var ChangeSet[NodeKind]
 
-proc sizeAndAlignment(tree; n): tuple[size, align: uint32] =
+proc sizeAndAlignment(c: Context; tree; n): tuple[size, align: uint32] =
   case tree[n].kind
   of Type:
-    sizeAndAlignment(tree, tree.child(tree.child(0), tree[n].val))
+    sizeAndAlignment(c, tree, tree.child(tree.child(0), tree[n].val))
   of Union, Record, Array:
     (tree[n, 0].val, tree[n, 1].val)
   of Int, UInt, Float:
     (tree[n].val, tree[n].val)
+  of Ptr:
+    (c.ptrSize, c.ptrSize)
   else:
     unreachable()
 
@@ -81,12 +84,12 @@ proc lowerGlobal(c; tree; n; bu) =
   let data = tree.child(n, 1)
 
   assert tree[data].kind == Data
-  let (s, a) = sizeAndAlignment(tree, tree.child(data, 0))
+  let (s, a) = sizeAndAlignment(c, tree, tree.child(data, 0))
   if tree.len(data) == 1:
     # (Data <typ>) -> (Data <align> <size>)
     bu.replace(n, bu.buildTree(
       tree(GlobalDef,
-        node(UInt, c.ptrSize.uint32),
+        node(Node(kind: Ptr)),
         tree(Data,
           # TODO: use packed values once supported, so that the full integer
           #       range works
@@ -96,15 +99,15 @@ proc lowerGlobal(c; tree; n; bu) =
     # (Data <typ> <content>) -> (Data <align> <content>)
     bu.replace(n, bu.buildTree(
       tree(GlobalDef,
-        node(UInt, c.ptrSize.uint32),
+        node(Node(kind: Ptr)),
         tree(Data,
           node(Immediate, a),
           node(tree[data, 1]))))) # the string value stays
 
-proc lower*(tree; ptrSize: int): ChangeSet[NodeKind] =
+proc lower*(tree; ptrSize: Positive): ChangeSet[NodeKind] =
   ## Computes the changeset representing the lowering for a whole module
   ## (`tree`). `ptrSize` is the size-in-bytes of a pointer.
-  var c = Context(ptrSize: ptrSize)
+  var c = Context(ptrSize: ptrSize.uint32)
   let (_, globals, procs) = tree.triplet(NodeIndex(0))
 
   for i, it in tree.pairs(globals):

passes/pass_legalizeBlobOps.nim

@@ -19,7 +19,7 @@ proc isBlob(tree; n): bool =
   case tree[n].kind
   of Blob: true
   of Type: tree[tree.child(0), tree[n].val].kind == Blob
-  of Int, UInt, Float: false
+  of Int, UInt, Float, Ptr: false
   else:    unreachable()
 
 proc prepareOperand(tree; n; bu): VirtualTree =

passes/pass_localsToBlob.nim

@@ -9,6 +9,8 @@ type
   LocalId = distinct uint32
 
   Context = object
+    ptrSize: int
+    # per-procedure context:
     locals: NodeIndex
     marker: PackedSet[LocalId]
 
@@ -32,9 +34,18 @@ proc resolve(tree; n: NodeIndex): NodeIndex =
 proc typeof(c; tree; id: LocalId): NodeIndex =
   tree.child(c.locals, ord id)
 
+proc sizeAndAlignment(c; tree; n): (uint32, uint32) =
+  case tree[n].kind
+  of Int, UInt, Float:
+    (tree[n].val, tree[n].val)
+  of Ptr:
+    (c.ptrSize.uint32, c.ptrSize.uint32)
+  else:
+    unreachable()
+
 proc isSimple(tree; n): bool =
   case tree[n].kind
-  of Int, UInt, Float: true
+  of Int, UInt, Float, Ptr: true
   of Blob: false
   of Type: isSimple(tree, tree.child(tree.child(0), tree[n].val))
   else:    unreachable()
@@ -111,18 +122,18 @@ proc lowerProc(c; tree; n; bu) =
     for it in c.marker.items:
       let
         slot = tree.child(c.locals, ord it)
-        size = tree[tree.resolve(slot)].val
+        (size, align) = sizeAndAlignment(c, tree, tree.resolve(slot))
       bu.replace slot:
-        bu.buildTree tree(Blob, node(Immediate, size), node(Immediate, size))
+        bu.buildTree tree(Blob, node(Immediate, size), node(Immediate, align))
 
     for it in newLocals.items:
       bu.insert m, tree.fin(c.locals), it
 
-proc lower*(tree): ChangeSet[NodeKind] =
+proc lower*(tree; ptrSize: Positive): ChangeSet[NodeKind] =
   ## Computes the changeset representing the lowering for a whole module
-  ## (`tree`). `ptrSize` is the size-in-bytes of a pointer value.
+  ## (`tree`).
 
-  var c = Context()
+  var c = Context(ptrSize: ptrSize)
   # lower the procedures:
   for it in tree.items(tree.child(2)):
     if tree[it].kind == ProcDef: