cda-group · segeljakt · Aug 16, 2023
diff --git a/.github/Dockerfile b/.github/Dockerfile
@@ -0,0 +1,26 @@
+FROM ubuntu:18.04
+
+# Add repos
+RUN sudo apt update
+RUN sudo apt install software-properties-common
+RUN add-apt-repository ppa:avsm/ppa
+
+RUN sudo apt update
+RUN sudo apt install -y clang libssl-dev ninja-build wget make curl opam
+
+# Install OCaml
+RUN opam init -y --disable-sandboxing
+RUN eval $(opam config env)
+RUN opam install -y dune core menhir
+
+# Install CMAKE
+RUN wget https://github.com/Kitware/CMake/releases/download/v3.23.1/cmake-3.23.1.tar.gz
+RUN tar -xf cmake-3.23.1.tar.gz
+RUN cd cmake-3.23.1
+RUN ./configure
+RUN make
+RUN make install
+
+# Install Rust
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \
+      sh -s -- --default-toolchain nightly -y
diff --git a/.github/continuous-integration.sh b/.github/continuous-integration.sh
@@ -58,8 +58,8 @@ function run-mlir-tests {
     run-step ninja -C $ARC_MLIR_BUILD/llvm-build/ check-arc-mlir
 }
 
-function run-runtime-tests {
-    cd arc-runtime
+function run-lang-tests {
+    cd arc-lang
     run-step arc-cargo "$@"
 }
 
@@ -123,8 +123,8 @@ case $1 in
   ;;
 
     cargo)
-	# We assume this is a arc-runtime cargo command line
+	# We assume this is a arc-sys cargo command line
 	shift
-	run-runtime-tests "$@"
+	run-sys-tests "$@"
 	;;
 esac
diff --git a/.github/workflows/arc-mlir-broken-links.yml b/.github/workflows/arc-mlir-broken-links.yml
@@ -11,7 +11,7 @@ jobs:
     steps:
 
     - name: Checkout source
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
 
     - name: Update submodule
       run: git submodule update --force --init --recursive

diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml
@@ -15,11 +15,11 @@ env:
 
 jobs:
   build-deploy:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
 
     steps:
     - name: Checkout source
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
 
     - name: Update apt index
       run: sudo apt-get update
@@ -29,91 +29,93 @@ jobs:
     - name: Install Cargo nightly
       uses: actions-rs/toolchain@v1
       with:
-          toolchain: nightly
+          toolchain: stable
           override: true
 
-    - name: Cache arc-preprocessor
-      id: cache-arc-preprocessor
-      uses: actions/cache@v2
+    - name: Cache mdbook-arc-lang
+      id: cache-mdbook-arc-lang
+      uses: actions/cache@v3
       with:
-        path: arc-docs/arc-preprocessor/target
-        key: ${{ runner.os }}-cargo-${{ hashFiles('arc-docs/arc-preprocessor/**') }}
+        path: ~/.cargo/bin/mdbook-arc-lang
+        key: ${{ runner.os }}-${{ hashFiles('arc-lang/crates/book/mdbook-arc-lang/**') }}
+
+    - name: Cache mdbook-grammar
+      id: cache-mdbook-grammar
+      uses: actions/cache@v3
+      with:
+        path: ~/.cargo/bin/mdbook-grammar
+        key: ${{ runner.os }}-${{ hashFiles('arc-lang/crates/book/mdbook-grammar/**') }}
+
+    - name: Cache mdbook-exec
+      id: cache-mdbook-exec
+      uses: actions/cache@v3
+      with:
+        path: ~/.cargo/bin/mdbook-exec
+        key: ${{ runner.os }}-${{ hashFiles('arc-lang/crates/book/mdbook-exec/**') }}
 
     - name: Cache mdbook-bib
       id: cache-mdbook-bib
-      uses: actions/cache@v2
+      uses: actions/cache@v3
       with:
         path: ~/.cargo/bin/mdbook-bib
-        key: ${{ runner.os }}-cargo-${{ env.MDBOOK_BIB_VERSION }}
+        key: ${{ runner.os }}-${{ env.MDBOOK_BIB_VERSION }}
 
     - name: Cache mdbook-linkcheck
       id: cache-mdbook-linkcheck
-      uses: actions/cache@v2
+      uses: actions/cache@v3
       with:
         path: ~/.cargo/bin/mdbook-linkcheck
-        key: ${{ runner.os }}-cargo-${{ env.MDBOOK_LINKCHECK_VERSION }}
+        key: ${{ runner.os }}-${{ env.MDBOOK_LINKCHECK_VERSION }}
 
     - name: Cache zola
       id: cache-zola
-      uses: actions/cache@v2
+      uses: actions/cache@v3
       with:
         path: ~/.cargo/bin/zola
-        key: ${{ runner.os }}-cargo-${{ env.ZOLA_VERSION }}
+        key: ${{ runner.os }}-${{ env.ZOLA_VERSION }}
 
-    - name: Build arc-preprocessor
-      if: steps.cache-arc-preprocessor.outputs.cache-hit != 'true'
-      run: cargo build --manifest-path arc-docs/arc-preprocessor/Cargo.toml
+    - name: Build mdbook-arc-lang
+      if: steps.cache-mdbook-arc-lang.outputs.cache-hit != 'true'
+      run: cargo install --path arc-lang/crates/book/mdbook-arc-lang
+
+    - name: Build mdbook-grammar
+      if: steps.cache-mdbook-grammar.outputs.cache-hit != 'true'
+      run: cargo install --path arc-lang/crates/book/mdbook-grammar
+
+    - name: Build mdbook-exec
+      if: steps.cache-mdbook-exec.outputs.cache-hit != 'true'
+      run: cargo install --path arc-lang/crates/book/mdbook-exec
 
     - name: Build mdbook-bib
       if: steps.cache-mdbook-bib.outputs.cache-hit != 'true'
-      run: cargo install mdbook-bib --version=${{ env.MDBOOK_BIB_VERSION }}
+      run: cargo install mdbook-bib --version=${{ env.MDBOOK_BIB_VERSION }} --features="log/max_level_warn"
 
     - name: Build mdbook-linkcheck
       if: steps.cache-mdbook-linkcheck.outputs.cache-hit != 'true'
       run: cargo install mdbook-linkcheck --version=${{ env.MDBOOK_LINKCHECK_VERSION }}
 
     - name: Build zola
       if: steps.cache-zola.outputs.cache-hit != 'true'
-      run: |
-        git clone https://github.com/getzola/zola.git
-        git -C zola checkout ad6c83412391dd748d4a9d8f3a0568687e73a447
-        cargo install --path=zola --version=${{ env.ZOLA_VERSION }} zola
-
-    # ---- arc-report ----
-
-    - name: Install LaTeX dependencies
-      run: sudo apt-get install texlive-xetex latexmk texlive-bibtex-extra biber
-
-    - name: Build arc-report
-      run: make -C arc-report
-
-    # ---- arc-docs ----
-
-    - name: Setup mdbook
-      uses: peaceiris/actions-mdbook@v1
-      with:
-        mdbook-version: '0.4.15'
-
-    - name: Build arc-docs
-      run: make -C arc-docs build
+      run: cargo install --git https://github.com/getzola/zola.git \
+                         --rev ad6c83412391dd748d4a9d8f3a0568687e73a447 \
+                         --version=${{ env.ZOLA_VERSION }}
 
-    # ---- arc-website ----
+    # ---- website ----
 
-    - name: Move artifacts to arc-website
+    - name: Move book to website
       run: |
-        mkdir arc-website/static
-        mv arc-report/target/main.pdf arc-website/static/Arc-Report.pdf
-        mv arc-docs/target/html arc-website/static/docs
+        mkdir -p arc-lang/crates/website/zola/static
+        mv arc-lang/target/mdbook/target/html arc-lang/crates/website/zola/static/book
 
-    - name: Update arc-website theme submodule
+    - name: Update website theme submodule
       run: |
-        git submodule update --init arc-website/themes/adidoks
-        mkdir -p arc-website/templates
-        cp arc-website/themes/adidoks/templates/blog/page.html \
-           arc-website/templates/blog.html
+        git submodule update --init arc-lang/crates/website/zola/themes/adidoks
+        mkdir -p arc-lang/crates/website/zola/templates
+        cp arc-lang/crates/website/zola/themes/adidoks/templates/blog/page.html \
+           arc-lang/crates/website/zola/templates/blog.html
 
-    - name: Build arc-website
-      run: make -C arc-website build
+    - name: Build website
+      run: cargo run --manifest-path arc-lang/Cargo.toml --package=website
 
     - name: Deploy to gh-pages
       uses: peaceiris/actions-gh-pages@v3
@@ -123,5 +125,5 @@ jobs:
         github.ref == 'refs/heads/master'
       with:
         deploy_key: ${{ secrets.ACTIONS_DEPLOY_KEY }}
-        publish_dir: arc-website/target
+        publish_dir: arc-lang/target/website
         force_orphan: true
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -20,7 +20,7 @@ jobs:
         #   target: x86_64-apple-darwin
     steps:
     - name: Checkout source
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
       with:
         submodules: 'recursive'
 
@@ -44,7 +44,7 @@ jobs:
 
         mv README.md "$package"
         mv arc-mlir/build/llvm-build/bin/{arc,arc-lang,arc-mlir} "$package/bin"
-        mv {arc-runtime,arc-lang/stdlib} "$package/share"
+        mv {arc-lang/stdlib} "$package/share"
 
         tar cvf "$package.tar" "$package"
 

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -5,8 +5,6 @@ on:
     paths:
       - "arc-lang/**"
       - "arc-mlir/**"
-      - "arc-runtime/**"
-      - "arc-python/**"
       - "build"
       - ".github/workflows/test.yml"
     branches:
@@ -25,7 +23,7 @@ jobs:
 
       # Checkout branch which we want to test
     - name: Checkout source
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
 
       # Update LLVM dependency
     - name: Update submodule
@@ -41,29 +39,14 @@ jobs:
       env:
         ARC_MLIR_BUILD: ${{ format('{0}/build', runner.temp) }}
 
-    - name: Pre-build arc-runtime
-      run: ./.github/continuous-integration.sh cargo test
-      env:
-        ARC_MLIR_BUILD: ${{ format('{0}/build', runner.temp) }}
-
     - name: Run MLIR tests
       run: ./.github/continuous-integration.sh run-mlir-tests
       env:
         ARC_MLIR_BUILD: ${{ format('{0}/build', runner.temp) }}
 
-    - name: "Run arc-runtime test: clippy"
-      run: ./.github/continuous-integration.sh cargo clippy
-      env:
-        ARC_MLIR_BUILD: ${{ format('{0}/build', runner.temp) }}
-
     - name: "Ccache/Sccache statistics"
       run: ./.github/continuous-integration.sh check-ccache
 
-#    - name: "Run arc-runtime test: fmt -- -v --check"
-#      run: ./.github/continuous-integration.sh cargo fmt -- -v --check
-#      env:
-#        ARC_MLIR_BUILD: ${{ format('{0}/build', runner.temp) }}
-
     - name: Check for dirty tree
       run: ./check-for-dirty-tree
 

diff --git a/.gitignore b/.gitignore
@@ -163,3 +163,9 @@ book/
 # zola
 
 static/
+
+# JavaScript
+node_modules/
+
+arc-lang/examples/input/csv
+arc-lang/examples/input/jpg
diff --git a/.gitmodules b/.gitmodules
@@ -7,6 +7,6 @@
 [submodule "arc-mlir/corrosion"]
 	path = arc-mlir/src/corrosion
 	url = https://github.com/AndrewGaspar/corrosion.git
-[submodule "arc-website/themes/adidoks"]
-	path = arc-website/themes/adidoks
-	url = https://github.com/aaranxu/adidoks.git
+[submodule "arc-lang/crates/website/themes/juice"]
+	path = arc-lang/crates/website/themes/juice
+	url = https://github.com/huhu/juice.git
diff --git a/README.md b/README.md
@@ -11,47 +11,23 @@ OCaml (and dune), Rust (and cargo), and C++ (and CMake and Ninja).
 
 ## Examples
 
-A basic streaming word-count application can be written in functional-style as follows:
-```
-val wordcounts = lines
-  .flatmap(_.split(" "))
-  .keyby(_)
-  .window(
-    length = 10min,
-    stride = 3min
-  )
-  .count()
-```
-
-The same code can also be written using a more declarative, relational-style, syntax. This concept is borrowed from [Morel](https://github.com/julianhyde/morel) and applied to streaming data.
+A streaming word-count application can be implemented in Arc-Lang as follows.
 
 ```
-val wordcounts =
-  from
-    line in lines,
-    word in line.split(" ")
-  keyby word
-  window
-    length = 10min
-    stride = 3min
-  reduce count
-    identity 1;
+def main() =
+    from line: String in source(topic: "text"),
+         word in line.split(" ") {
+        group word
+        window count as w {
+            length 10min
+            step 3min
+            compute count
+        }
+        select {word, w.count}
+        into sink(topic: "wordcount")
+    }
 ```
 
-## Feature highlights
-
-* Statically typed with global type inference.
-* Parametric polymorphism (generics and rows) and ad-hoc polymorphism (type classes).
-* Mix of functional syntax, imperative control-flow/mutation, and relational operators.
-* Algebraic data types.
-* First-class data streams.
-* Complex event processing using tasks.
-* Window-based computation.
-* Low-level compilation and distributed execution.
-* Command-line interface for data ingestion.
-
-Note: All features have not yet been implemented :)
-
 ## Installation
 
 ```bash
@@ -71,13 +47,7 @@ git submodule update --init --recursive
 
 * [`arc-lang`](https://github.com/cda-group/arc/tree/master/arc-lang) - A compiler for Arc-Lang.
 * [`arc-mlir`](https://github.com/cda-group/arc/tree/master/arc-mlir) - An optimizer for Arc-Lang.
-* [`arc-runtime`](https://github.com/cda-group/arc/tree/master/arc-runtime) - A local runtime which supports the execution of Arc-Lang programs.
-* [`arc-python`](https://github.com/cda-group/arc/tree/master/arc-python) - A Python library for writing Arc-Lang applications.
-
-## Related Projects
-
-* [`arcon`](https://github.com/cda-group/arcon) - A distributed runtime which will support execution of Arc-Lang.
-* [`kompact`](https://github.com/kompics/kompact) - A component-actor middleware which Arc-Runtime and Arcon are both implemented in.
+* [`arc-sys`](https://github.com/cda-group/arc/tree/master/arc-sys) - A distributed system for executing Arc-Lang programs.
 
 ## Other
 

diff --git a/arc-docs/.gitignore b/arc-docs/.gitignore
diff --git a/arc-docs/Makefile b/arc-docs/Makefile