Add dev container, fix bugs in compiler setup (#20)

Features added:
* Add clang include directory to cfg
* Adding statiging release
* Add dev container
* Add runtime library interface
* Adding first version of unistd.c
* Adding print for exit code at debug exit

Bug Fixes: 
* Fix bugs in compiler cfg template
* Fix namecmp to recognize substrings as non equal
* Fix abi thread local issue, add test program build to top level cmake project
* Fix makro in cmake template
* Fix debug to handle unlimited number of input items or input-/outputsets

Author: tom-kuchler

.github/workflows/build_check.yml renamed to .github/workflows/release_experimental.yml

@@ -1,8 +1,8 @@
-name: make sure it still builds before merging into main 
+name: release latest with all library versions
 
 on:
   pull_request:
-    branches: [ "main" ]
+    branches: [ "dev/staging" ]
 
 jobs:
   build:
@@ -57,4 +57,15 @@ jobs:
       run: cmake --build ${{ steps.build.outputs.dir }}
 
     - name: Install
-      run: cmake --install ${{ steps.build.outputs.dir }} 
+      run: cmake --install ${{ steps.build.outputs.dir }}
+
+    - name: Compress
+      shell: bash
+      working-directory: ${{ steps.build.outputs.dir }}
+      run: tar -czf "dandelion_sdk_${{matrix.build}}_${{matrix.platform}}_${{matrix.cpu}}.tar.gz" dandelion_sdk
+    
+    - name: Add to release
+      shell: bash
+      run: gh release upload experimental "${{ steps.build.outputs.dir }}/dandelion_sdk_${{matrix.build}}_${{matrix.platform}}_${{matrix.cpu}}.tar.gz" --clobber
+      env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

CMakeLists.txt

@@ -15,6 +15,7 @@ option(NEWLIB "toggles builing of newlib on top of dandelion interface" OFF)
 # string options
 set(ARCHITECTURE ${CMAKE_SYSTEM_PROCESSOR} CACHE STRING "the architecture to build for")
 set(DANDELION_PLATFORM "debug" CACHE STRING "backend to build for")
+set(PAGE_SIZE "4096" CACHE STRING "Page size for the runtime to assume.")
 message(STATUS "Building for ${DANDELION_PLATFORM} on ${ARCHITECTURE}")
 
 # Constants
@@ -119,6 +120,33 @@ if(NEWLIB)
                 <INSTALL_DIR>/${ARCHITECTURE}-dandelion/lib
                 ${CMAKE_INSTALL_PREFIX}/lib 
     )
+    # Create folder at configuration time to avoid issues with includes
+    file(MAKE_DIRECTORY ${CMAKE_INSTALL_PREFIX}/include)
+    file(MAKE_DIRECTORY ${CMAKE_INSTALL_PREFIX}/include/sys)
+
+    add_library(dlibc INTERFACE IMPORTED)
+    target_compile_options(dlibc INTERFACE 
+        -D_GNU_SOURCE=1
+        -D__GNU__
+        -D__rtems__
+    )
+    target_include_directories(dlibc INTERFACE
+        ${CMAKE_INSTALL_PREFIX}/include
+        ${CMAKE_INSTALL_PREFIX}/include/sys
+    )
+    target_link_directories(dlibc INTERFACE ${CMAKE_INSTALL_PREFIX}/lib)
+    target_link_libraries(dlibc INTERFACE
+        ${CMAKE_INSTALL_PREFIX}/lib/libc.a
+        ${CMAKE_INSTALL_PREFIX}/lib/libg.a
+        ${CMAKE_INSTALL_PREFIX}/lib/libm.a
+    )
+    target_link_options(dlibc INTERFACE
+        -T${CMAKE_CURRENT_SOURCE_DIR}/sdk_install/linker.ld
+        -fuse-ld=lld
+        -static
+        -nostdlib
+    )
+    add_dependencies(dlibc newlib)
 
     set(LLVM_C_FLAGS "-D_GNU_SOURCE=1")
     string(APPEND LLVM_C_FLAGS " -D_POSIX_TIMERS")
@@ -191,13 +219,37 @@ if(NEWLIB)
             -DLIBCXXABI_BAREMETAL=ON
             -DLIBCXXABI_ENABLE_SHARED=OFF
             -DLIBCXXABI_USE_LLVM_UNWINDER=ON
+            -DLIBCXXABI_ENABLE_THREADS=OFF
             -DLIBCXX_ENABLE_SHARED=OFF
             -DLIBCXX_ENABLE_STATIC=ON
             -DLIBCXX_INCLUDE_BENCHMARKS=OFF
             -DLIBCXX_CXX_ABI=libcxxabi
             -DLIBCXX_ENABLE_FILESYSTEM=ON
             -DLIBCXX_ENABLE_RANDOM_DEVICE=OFF
+            -DLIBCXX_HAS_PTHREAD_API=ON
     )
     add_dependencies(llvmproject newlib)
 
+    # Create folder at configuration time to avoid issues with includes
+    file(MAKE_DIRECTORY ${CMAKE_INSTALL_PREFIX}/include/c++/v1)
+
+    add_library(dlibcxx INTERFACE IMPORTED)
+    target_include_directories(dlibcxx INTERFACE
+        "${CMAKE_INSTALL_PREFIX}/include/c++/v1"
+    )
+    target_link_libraries(dlibcxx INTERFACE
+        "${CMAKE_INSTALL_PREFIX}/lib/libc++.a"
+        "${CMAKE_INSTALL_PREFIX}/lib/libc++abi.a"
+        "${CMAKE_INSTALL_PREFIX}/lib/libunwind.a"
+    )
+    add_dependencies(dlibcxx llvmproject)
+
+    add_library(runtime INTERFACE IMPORTED)
+    target_link_libraries(runtime INTERFACE
+        "${CMAKE_INSTALL_PREFIX}/lib/generic/libclang_rt.builtins-${ARCHITECTURE}.a"
+    )
+    add_dependencies(runtime llvmproject)
+
+    add_subdirectory(test_programs)
+
 endif()

Dockerfile

@@ -0,0 +1,34 @@
+# syntax=docker/dockerfile:1
+FROM ubuntu:24.04
+
+# arguments
+ARG BUILD="Release"
+ARG PLATFORM="debug"
+ARG TARGET_ARCH="x86_64"
+ARG VERSION="experimental"
+
+# install packages required packages to get things running
+RUN apt update
+RUN apt-get install -y wget gpg
+RUN apt-get install -y lsb-release
+RUN apt-get install -y software-properties-common
+RUN apt-get install -y gcc g++ # clang uses gcc to interface with the linker
+
+# download initial clang
+RUN wget https://apt.llvm.org/llvm.sh -O ~/llvm.sh
+RUN chmod +x ~/llvm.sh
+RUN ~/llvm.sh
+RUN ln -s /usr/bin/ld.lld-19 /usr/bin/ld.lld
+RUN rm -r /usr/include/*
+
+# download and set up dandelionSDK
+RUN wget https://github.com/eth-easl/dandelionSDK/releases/download/${VERSION}/dandelion_sdk_${BUILD}_${PLATFORM}_${TARGET_ARCH}.tar.gz -O ~/dandelionSDK.tar.gz
+RUN tar -xzf ~/dandelionSDK.tar.gz -C ~/
+RUN ~/dandelion_sdk/create-compiler.sh -d -c clang-19
+
+ENV CC="clang"
+ENV CXX="clang++"
+
+# additional useful tooling
+RUN apt-get install -y cmake
+RUN apt-get install -y git

README.md

@@ -30,6 +30,12 @@ dumping outputs to the terminal when terminating.
 Fixing it is on our agenda, but not of highest priority.
 If you want to use Dandelion and need C++ support for aarch64, please get in touch and we may be able to provide experimental builds or bump the priority to get it fixed.
 
+### Other parameters
+
+Additionally, the following parameters can be used to influence system behaviour.
+
+- `PAGE_SIZE` sets the page size the runtime will assume to optimize allocation etc. and makes available to others to depend on. Should be a multiple of sizeof(size_t) for the platform compiled for.
+
 ## Interface expectations
 ### libc
 When using libc or any system on top of it values can be fed into stdin, argv and environ by specifying a input set called "stdio".
@@ -97,6 +103,28 @@ For newlib to be built correctly the autoconf version 2.69.
 This also enables the build of the in memory file system,
 which can also be built without the other newlib builds by setting `-DDANDELION_FS=ON`
 
+## Build Container
+We also provide a docker file to construct a build container with the correct tools set up in them.
+To create the container with docker use the following command:
+```
+docker build -t dandelion_dev_docker . [--no-cache]
+```
+Use the `--no-cache` if you want to rebuild the container from scratch.
+
+To use the container use the following command:
+```
+docker run --rm -it --mount type=bind,src=<path to your source folder>,dst=/workspace --workdir=/workspace dandelion_dev_docker:latest bash
+``` 
+You will enter the container and your source folder will be mounted at `/workspace`.
+The default clang and clang++ are set up to compile for dandelion debug on the machine you are running.
+If you want a different target, you can set build, platform and architecture in the docker file.
+Files you build in the container will be visible outside, as long as they are in the workspace directory.
+
+To add more tools to the build environment, they can be added to the docker file, or you can base your container build on our docker file by importing it with:
+```
+FROM dandelion_dev_docker:latest
+```
+
 ## Freestanding
 The GCC/Clang standard expects 4 functions to allways be provided in any environment (even freestanding), which allow the compiler to always just insert them.
 (https://gcc.gnu.org/onlinedocs/gcc/Standards.html)

file_system/file_system.c

@@ -16,7 +16,7 @@ D_File *create_file(Path *name, char *content, size_t length, uint32_t mode) {
   if (new_file == NULL) {
     return NULL;
   }
-  if (name->length > FS_NAME_LENGHT) {
+  if (name->length > FS_NAME_LENGTH) {
     return NULL;
   }
   memcpy(new_file->name, name->path, name->length);
@@ -43,7 +43,7 @@ D_File *create_file(Path *name, char *content, size_t length, uint32_t mode) {
 
 D_File *create_directory(Path *name, uint32_t mode) {
   D_File *new_file = dandelion_alloc(sizeof(D_File), _Alignof(D_File));
-  if (name->length > FS_NAME_LENGHT) {
+  if (name->length > FS_NAME_LENGTH) {
     return NULL;
   }
   memcpy(new_file->name, name->path, name->length);
@@ -69,12 +69,12 @@ int link_file_to_folder(D_File *folder, D_File *file) {
     file->next = NULL;
   } else {
     D_File *current = folder->child;
-    if (namecmp(file->name, current->name, FS_NAME_LENGHT) < 0) {
+    if (namecmp(file->name, FS_NAME_LENGTH, current->name, FS_NAME_LENGTH) < 0) {
       file->next = current;
       folder->child = file;
     } else {
       while (current->next != NULL) {
-        if (namecmp(file->name, current->next->name, FS_NAME_LENGHT) < 0) {
+        if (namecmp(file->name, FS_NAME_LENGTH, current->next->name, FS_NAME_LENGTH) < 0) {
           break;
         } else {
           current = current->next;
@@ -98,7 +98,7 @@ D_File *find_file_in_dir(D_File *directory, Path file) {
   for (D_File *current = directory->child; current != NULL;
        current = current->next) {
     int cmp_result =
-        namecmp(current->name, file.path, MIN(file.length, FS_NAME_LENGHT));
+        namecmp(current->name, FS_NAME_LENGTH, file.path, file.length);
     if (cmp_result == 0) {
       return current;
     } else if (cmp_result > 0) {
@@ -138,7 +138,7 @@ D_File *create_directories(D_File *directory, Path path, char prevent_up) {
   }
   for (Path current_path = get_component_advance(&path);
        current_path.length > 0; current_path = get_component_advance(&path)) {
-    if (current_path.length > FS_NAME_LENGHT) {
+    if (current_path.length > FS_NAME_LENGTH) {
       return NULL;
     } else if (current_path.length == 1 && current_path.path[0] == '.') {
       // handle special case of single dot for current directory
@@ -483,7 +483,7 @@ int fs_initialize(int *argc, char ***argv, char ***environ) {
       return -1;
     }
     int is_stdio_folder =
-        namecmp(set_path.path, "stdio", MIN(set_path.length, 5));
+        namecmp(set_path.path, set_path.length, "stdio", 5);
     size_t input_items = dandelion_input_buffer_count(set_index);
     for (size_t item_index = 0; item_index < input_items; item_index++) {
       IoBuffer *item_buffer = dandelion_get_input(set_index, item_index);
@@ -510,19 +510,19 @@ int fs_initialize(int *argc, char ***argv, char ***environ) {
       }
       if (is_stdio_folder == 0) {
         int is_stdin =
-            namecmp(file_path.path, "stdin", MIN(file_path.length, 5));
+            namecmp(file_path.path, file_path.length, "stdin", 5);
         if (is_stdin == 0) {
           error = open_existing_file(STDIN_FILENO, item_file, O_RDONLY, 0, 0);
           if (error != 0)
             return error;
         }
-        int is_argv = namecmp(file_path.path, "argv", MIN(file_path.length, 4));
+        int is_argv = namecmp(file_path.path, file_path.length, "argv", 4);
         if (is_argv == 0) {
           setup_charpparray(item_buffer->data, item_buffer->data_len, argc,
                             argv);
         }
         int is_environ =
-            namecmp(file_path.path, "environ", MIN(file_path.length, 7));
+            namecmp(file_path.path, file_path.length, "environ", 7);
         if (is_environ == 0) {
           int envc;
           setup_charpparray(item_buffer->data, item_buffer->data_len, &envc,
@@ -588,7 +588,7 @@ int add_output_from_file(D_File *file, Path previous_path, size_t set_index) {
   switch (file->type) {
   case FILE:
     // check name length an create string with complete file name
-    name_length = namelen(file->name, FS_NAME_LENGHT);
+    name_length = namelen(file->name, FS_NAME_LENGTH);
     new_buffer = dandelion_alloc(previous_path.length + name_length, 1);
     if (new_buffer == NULL) {
       return -1;
@@ -631,7 +631,7 @@ int add_output_from_file(D_File *file, Path previous_path, size_t set_index) {
     return 0;
   case DIRECTORY:
     // check name length and create a new string / path to recurse further
-    name_length = namelen(file->name, FS_NAME_LENGHT);
+    name_length = namelen(file->name, FS_NAME_LENGTH);
     new_buffer = dandelion_alloc(previous_path.length + name_length + 1, 1);
     if (new_buffer == NULL) {
       dandelion_exit(ENOMEM);
@@ -677,10 +677,10 @@ int fs_terminate() {
     for (D_File *out_file = set_directory->child; out_file != NULL;
          out_file = out_file->next) {
       // ignore argv, environ and stdin in the stdio folder
-      if (namecmp(set_ident.path, "stdio", MIN(set_ident.length, 5)) == 0) {
-        if (namecmp(out_file->name, "environ", 7) == 0 ||
-            namecmp(out_file->name, "argv", 4) == 0 ||
-            namecmp(out_file->name, "stdin", 5) == 0)
+      if (namecmp(set_ident.path, set_ident.length, "stdio", 5) == 0) {
+        if (namecmp(out_file->name, FS_NAME_LENGTH, "environ", 7) == 0 ||
+            namecmp(out_file->name, FS_NAME_LENGTH, "argv", 4) == 0 ||
+            namecmp(out_file->name, FS_NAME_LENGTH, "stdin", 5) == 0)
           continue;
       }
       add_output_from_file(out_file, empty_path, set_index);

file_system/file_system.h

@@ -6,7 +6,7 @@
 #include <stdint.h>
 
 #ifndef FS_NAME_LENGTH
-#define FS_NAME_LENGHT 64
+#define FS_NAME_LENGTH 64
 #endif
 
 #ifndef FS_CHUNK_SIZE
@@ -40,7 +40,7 @@ typedef struct FileChunk {
 
 // Use D_File instead of File, to avoid potential naming overlap
 typedef struct D_File {
-  char name[FS_NAME_LENGHT];
+  char name[FS_NAME_LENGTH];
   struct D_File *next;
   struct D_File *parent;
   FileType type;

file_system/fs_implementation.c

@@ -125,7 +125,7 @@ int dandelion_open(const char *name, int flags, uint32_t mode) {
     Path total_path = path_from_string(name);
     Path dir_path = get_directories(total_path);
     Path file_name = get_file(total_path);
-    if (file_name.length >= FS_NAME_LENGHT) {
+    if (file_name.length >= FS_NAME_LENGTH) {
       return -EINVAL;
     }
     D_File *parent = create_directories(fs_root, dir_path, 0);
@@ -643,7 +643,7 @@ int dandelion_readdir(DIR *directory, struct dirent *dirent) {
     return -1;
   }
   directory->child++;
-  size_t max_name_length = MIN(256, FS_NAME_LENGHT);
+  size_t max_name_length = MIN(256, FS_NAME_LENGTH);
   memcpy(dirent->d_name, current_child->name, max_name_length - 1);
   dirent->d_name[max_name_length] = 0;
   dirent->d_ino = 0;

file_system/paths.h

@@ -23,16 +23,24 @@ static inline size_t namelen(const char *const name, size_t max_len) {
   return length;
 }
 
-static inline int namecmp(const char *const name1, const char *const name2,
-                          size_t max_length) {
+static inline int namecmp(const char *const name1, size_t name1_length, const char *const name2,
+                          size_t name2_length) {
+  size_t max_length = name1_length < name2_length ? name1_length : name2_length;
   for (size_t index = 0; index < max_length; index++) {
+    // this also automatically returns -1 if one of them is null terminated earlier than their length
     if (name1[index] != name2[index])
       return name1[index] < name2[index] ? -1 : 1;
     // are the same if we got here
     if (name1[index] == '\0')
       return 0;
   }
-  return 0;
+  // they are the same until the end of the shorter one or there has not been null termination
+  if(name1_length < name2_length)
+    return -1;
+  else if(name2_length < name1_length)
+    return 1;
+  else 
+    return 0;
 }
 
 Path path_from_string(const char *const str);

newlib_shim/Makefile.inc

@@ -2,4 +2,5 @@ libc_a_SOURCES += \
     %D%/dirent.c \
     %D%/pthread.c \
     %D%/shim.c \
-    %D%/time.c
+    %D%/time.c \
+    %D%/unistd.c

newlib_shim/patch_script

@@ -22,6 +22,7 @@ cp $THIS_DIR/shim.c $1/newlib/libc/sys/dandelion/shim.c
 cp $THIS_DIR/dirent.c $1/newlib/libc/sys/dandelion/dirent.c
 cp $THIS_DIR/pthread.c $1/newlib/libc/sys/dandelion/pthread.c
 cp $THIS_DIR/time.c $1/newlib/libc/sys/dandelion/time.c
+cp $THIS_DIR/unistd.c $1/newlib/libc/sys/dandelion/unistd.c
 cp $THIS_DIR/../include/dandelion/crt.h $1/newlib/libc/sys/dandelion/crt.h
 if ! test -d $1/newlib/libc/sys/dandelion/sys
 then