Collaboration

.dockerignore

@@ -0,0 +1,3 @@
+*.csv
+*.csv2
+.git

.gitignore

@@ -0,0 +1,4 @@
+extras/model.go
+testdata/*
+*.gz
+lambdadb

Dockerfile

@@ -5,7 +5,7 @@ RUN apk update && apk add --no-cache git
 RUN apk --no-cache add ca-certificates
 
 WORKDIR /app
-COPY . /app/
+COPY *.go /app/
 
 # Fetch dependencies.
 RUN go get -d -v
@@ -23,6 +23,7 @@ COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
 #COPY --from=builder /app/files/ITEMS.txt.gz /app/files/ITEMS.txt.gz
 
 WORKDIR /app
+
 # Run the binary.
 
 ENV http_db_host "0.0.0.0:8000"

README.md

@@ -1,32 +1,78 @@
 # LambdaDB
 In memory database that uses filters to get the data you need.
+Lambda DB has a tiny codebase which does a lot
+Lambda is not ment as a persistance storage or a replacement for a traditional
+Database but as fast analytics engine cache representation engine.
 
-Can be used for your needs by changing the models.go file to your needs.
+powers: https://dego.vng.nl
+
+## Properties:
+
+- Insanely fast API. 1ms respsonses
+- Fast to setup.
+- Easy to deploy.
+- Easy to customize.
+- Easy export data
+
+- Implement custom authorized filters.
+
+## Indexes
+
+- S2 geoindex for fast point lookup
+- Bitarrays
+- Mapping
+
+- Your own special needs indexes!
+
+## Flow:
+
+Generate a model and load your data.
+The API is generated from your model.
+Deploy.
+
+Condition: Your dataset must fit in memory.
+
+Can be used for your needs by changing the `models.go` file to your needs.
 Creating and registering of the functionality that is needed.
 
 
 ### Steps
 You can start the database with only a csv.
 Go over steps below, And see the result in your browser.
+
 1. place csv file, in dir extras.
-2. `python3 create_model.py > ../model.go`
-3. cd ../
-4. go fmt
+2. `python3 create_model_.py`  answer the questions.
+3. go fmt model.go
+4. mv model.go ../
 5. go build
 6. ./lambda --help
 7. ./lambda  --csv assets/items.csv or `python3 ingestion.py -b 1000`
 9. curl 127.0.0.1:8128/help/
 10. browser 127.0.0.1:8128/
 
-
 11. instructions curl 127.0.0.1:8128/help/ | python -m json.tool
 
 
-
 ### Running
 
 sudo docker-compose up  --no-deps --build
 
 promql {instance="lambdadb:8000"}
 
 python3 extras/ingestion.py  -f movies_subset.tsv -format tsv -dbhost 127.0.0.1:8000
+=======
+
+1. instructions curl 127.0.0.1:8000/help/ | python -m json.tool
+
+### Questions
+
+
+
+### TODO
+
+- load data directly from a database (periodic)
+- document the `create_model.py` questions
+- use a remote data source
+- use some more efficient storage method (done)
+- generate swagger API
+- Add more tests

column.go

@@ -0,0 +1,148 @@
+package main
+
+import (
+	"errors"
+	"fmt"
+	"github.com/Workiva/go-datastructures/bitarray"
+	"log"
+	"strings"
+)
+
+type fieldIdxMap map[string]uint32
+type IdxFieldMap map[uint32]string
+
+/*
+MappedColumn store fields in Idx
+*/
+
+type MappedColumn struct {
+	Idx        fieldIdxMap // stores field to int values
+	Field      IdxFieldMap // stores int to field values to recover actual fields
+	IdxTracker uint32
+	Name       string
+}
+
+type ColumnRegister map[string]MappedColumn
+
+var RegisteredColumns ColumnRegister
+
+func init() {
+	RegisteredColumns = make(ColumnRegister)
+}
+
+func NewReapeatedColumn(column string) MappedColumn {
+	m := MappedColumn{
+		make(fieldIdxMap),
+		make(IdxFieldMap),
+		0,
+		column,
+	}
+	RegisteredColumns[column] = m
+	return m
+}
+
+// Store field name as idx value and idx as field value
+func (m *MappedColumn) Store(field string) {
+
+	if _, ok := m.Idx[field]; !ok {
+		m.Idx[field] = m.IdxTracker
+		m.Field[m.IdxTracker] = field
+		m.IdxTracker++
+	}
+}
+
+// Store Array field (postgres Array).
+func (m *MappedColumn) StoreArray(field string) []uint32 {
+
+	fieldsArray := make([]uint32, 0)
+
+	// parsing {a, b} array values
+	// string should be at least 2 example "{}" == size 2
+	if len(field) > 2 {
+		fields, err := ParsePGArray(field)
+
+		if err != nil {
+			log.Fatal(err, "error parsing array ")
+		}
+
+		for _, gd := range fields {
+			m.Store(gd)
+		}
+
+		for _, v := range fields {
+			fieldsArray = append(fieldsArray, Gebruiksdoelen.GetIndex(v))
+		}
+	}
+	return fieldsArray
+}
+
+func (m *MappedColumn) GetValue(idx uint32) string {
+	return m.Field[idx]
+}
+
+func (m *MappedColumn) GetArrayValue(idxs []uint32) string {
+
+	result := make([]string, 0)
+	for _, v := range idxs {
+		vs := m.GetValue(v)
+		result = append(result, vs)
+	}
+	return strings.Join(result, ", ")
+}
+
+func (m *MappedColumn) GetIndex(s string) uint32 {
+	return m.Idx[s]
+}
+
+// SetBitArray
+func SetBitArray(column string, i uint32, label int) {
+
+	var ba bitarray.BitArray
+	var ok bool
+
+	// check if map of bitmaps is present for column
+	var map_ba fieldBitarrayMap
+
+	if _, ok = BitArrays[column]; !ok {
+		map_ba := make(fieldBitarrayMap)
+		BitArrays[column] = map_ba
+	}
+
+	map_ba = BitArrays[column]
+
+	// check for existing bitarray for i value
+	ba, ok = map_ba[i]
+
+	if !ok {
+		ba = bitarray.NewSparseBitArray()
+		map_ba[i] = ba
+	}
+	// set bit for item label.
+	ba.SetBit(uint64(label))
+}
+
+func GetBitArray(column, value string) (bitarray.BitArray, error) {
+
+	var ok bool
+
+	if _, ok = BitArrays[column]; !ok {
+		return nil, errors.New("no bitarray filter found for " + column)
+	}
+
+	// convert string value to actual indexed int.
+	i, ok := RegisteredColumns[column].Idx[value]
+
+	if !ok {
+		msg := fmt.Sprintf("no indexed int value found for %s %s", column, value)
+		return nil, errors.New(msg)
+	}
+
+	ba, ok := BitArrays[column][i]
+
+	if !ok {
+		msg := fmt.Sprintf("no bitarray found for %s %s %d", column, value, i)
+		return nil, errors.New(msg)
+	}
+
+	return ba, nil
+}